file restructuring for sdk

WARNING: This breaks the sparkd clang build (only works with gcc)

.devcontainer/Dockerfile

@@ -0,0 +1,122 @@
+FROM ubuntu:22.04
+
+ARG DEBIAN_FRONTEND=noninteractive
+
+# Note installations that take a while (more than 10s)
+
+RUN apt-get update
+RUN apt-get install -y ca-certificates
+RUN apt-get install -y ccache
+RUN apt-get install -y curl
+RUN apt-get install -y fish
+RUN apt-get install -y gdb
+# git-all: ~250s
+RUN apt-get install -y git-all
+RUN apt-get install -y gnupg
+# liblttng-ust-dev: ~30s
+RUN apt-get install -y liblttng-ust-dev
+RUN apt-get install -y libssh2-1-dev
+RUN apt-get install -y locales
+RUN apt-get install -y ninja-build
+# openssh-server: ~40s
+RUN apt-get install -y openssh-server
+RUN apt-get install -y pkg-config
+RUN apt-get install -y software-properties-common
+RUN apt-get install -y sudo
+RUN apt-get install -y tzdata
+RUN apt-get install -y vim
+RUN apt-get install -y zip
+
+# Python: ~30s
+RUN apt-get install -y \
+   python3-pip \
+   python3-venv \
+   && python3 -m pip install --upgrade pip
+
+# GCC
+ARG gcc_major=13
+RUN \
+   add-apt-repository ppa:ubuntu-toolchain-r/test \
+   && apt-get update \
+   && apt-get install -y g++-$gcc_major
+
+# Clang
+ARG clang_major=17
+RUN \
+   curl -fsSL https://apt.llvm.org/llvm-snapshot.gpg.key 2> /dev/null \
+   | gpg --dearmor - \
+   | tee /usr/share/keyrings/llvm-snapshot.gpg > /dev/null \
+   && echo "deb [signed-by=/usr/share/keyrings/llvm-snapshot.gpg] http://apt.llvm.org/$(lsb_release -cs)/ llvm-toolchain-$(lsb_release -cs)-$clang_major main" \
+   | tee -a "/etc/apt/sources.list.d/llvm.list" \
+   && apt-get update \
+   && apt-get install -y \
+   clang-$clang_major \
+   clang-format-$clang_major \
+   clangd-$clang_major
+
+# CMake
+RUN \
+   curl -fsSL https://apt.kitware.com/keys/kitware-archive-latest.asc 2> /dev/null \
+   | gpg --dearmor - \
+   | tee /usr/share/keyrings/kitware-archive-keyring.gpg > /dev/null \
+   && echo "deb [signed-by=/usr/share/keyrings/kitware-archive-keyring.gpg] https://apt.kitware.com/ubuntu/ $(lsb_release -cs) main" \
+   | tee /etc/apt/sources.list.d/kitware.list > /dev/null \
+   && apt-get update \
+   && apt-get install -y cmake
+
+# Dotnet
+RUN apt-get update && apt-get install -y dotnet7
+
+# Node
+ARG node_major=18
+RUN mkdir -p /etc/apt/keyrings \
+   && curl -fsSL https://deb.nodesource.com/gpgkey/nodesource-repo.gpg.key \
+   | gpg --dearmor -o /etc/apt/keyrings/nodesource.gpg \
+   && echo "deb [signed-by=/etc/apt/keyrings/nodesource.gpg] https://deb.nodesource.com/node_$node_major.x nodistro main" \
+   | tee /etc/apt/sources.list.d/nodesource.list \
+   && apt-get update \
+   && apt-get install -y nodejs
+
+# Emumba (deps)
+RUN apt-get update && apt-get -y install rapidjson-dev
+# Emumba
+COPY emumba /emumba
+RUN dpkg -i /emumba/emumba-deribit-connector-3.3.1-Linux.LICENSE_DISABLED.deb \
+    && rm -r /emumba
+
+# Pcap
+RUN apt-get update && apt-get install -y libpcap0.8-dev
+
+# Set up default executables (update-alternatives)
+RUN \
+   update-alternatives --install /usr/bin/clang clang /usr/bin/clang-$clang_major 100; \
+   update-alternatives --install /usr/bin/clang++ clang++ /usr/bin/clang++-$clang_major 100; \
+   update-alternatives --install /usr/bin/clang-format clang-format /usr/bin/clang-format-$clang_major 100; \
+   update-alternatives --install /usr/bin/clangd clangd /usr/bin/clangd-$clang_major 100; \
+   update-alternatives --install /usr/bin/gcc gcc /usr/bin/gcc-$gcc_major 100; \
+   update-alternatives --install /usr/bin/g++ g++ /usr/bin/g++-$gcc_major 100
+
+# Add compiler symlinks for ccache 
+RUN \
+   ln -s /usr/bin/ccache /usr/local/bin/clang; \
+   ln -s /usr/bin/ccache /usr/local/bin/clang++; \
+   ln -s /usr/bin/ccache /usr/local/bin/clang-$clang_major; \
+   ln -s /usr/bin/ccache /usr/local/bin/clang++-$clang_major; \
+   ln -s /usr/bin/ccache /usr/local/bin/gcc; \
+   ln -s /usr/bin/ccache /usr/local/bin/g++
+
+RUN locale-gen en_US.UTF-8
+
+ARG docker_user="bts"
+ARG docker_uid="1000"
+ARG docker_pass="bts"
+
+RUN \
+   groupadd -g "$docker_uid" "$docker_user" \
+   && useradd \
+   -m \
+   -u "$docker_uid" \
+   -g "$docker_uid" \
+   -G sudo \
+   --password "$(openssl passwd -1 "$docker_pass")" \
+   "$docker_user"

.devcontainer/devcontainer.json

@@ -0,0 +1,48 @@
+{
+  "dockerFile": "Dockerfile",
+  // Creates local dir to mount to HOME in docker 
+  "initializeCommand": "mkdir -p ${localEnv:HOME}/.bts/devcontainer-mounts/${devcontainerId}",
+  "hostRequirements": {
+    "memory": "4gb",
+    "storage": "6gb"
+  },
+  "postCreateCommand": "/home/bts/spark/docker/sparkd/dev/devcontainer_post_create.sh",
+  "containerUser": "bts",
+  "workspaceMount": "source=${localWorkspaceFolder},target=/home/bts/spark,type=bind",
+  "workspaceFolder": "/home/bts/spark",
+  "updateRemoteUserUID": true,
+  "shutdownAction": "none",
+  // Note: If running multiple containers on the same host, each must use a
+  // unique set of ports. Usually we just add 1000 to each port as needed. For
+  // example, if 8000-8003 is taken, go with 9000-9003, then 10000-10003, etc.
+  "runArgs": [
+    "--shm-size",
+    "2gb",
+    "-p",
+    "8000:8000",
+    "-p",
+    "8001:8001",
+    "-p",
+    "8002:8002",
+    "-p",
+    "8003:8003"
+  ],
+  "mounts": [
+    "source=${localEnv:HOME}/.bts/devcontainer-mounts/${devcontainerId},target=/home/bts,type=bind"
+  ],
+  "customizations": {
+    "vscode": {
+      "settings": {
+        "clangd.arguments": [
+          "--compile-commands-dir=${workspaceFolder}/build/clang/debug"
+        ]
+      },
+      "extensions": [
+        "eamodio.gitlens",
+        "llvm-vs-code-extensions.vscode-clangd",
+        "ms-vscode.cmake-tools",
+        "ms-vscode.cpptools"
+      ]
+    }
+  }
+}

.devcontainer/emumba/install_emumba-deribit-connector.sh

@@ -0,0 +1,4 @@
+#!/usr/bin/env bash
+
+this_dir="$(realpath "$(dirname "$(readlink -f "${BASH_SOURCE[0]}")")")"
+sudo dpkg -i "$this_dir/emumba-deribit-connector-3.3.1-Linux.LICENSE_DISABLED.deb"

.gitignore

@@ -54,6 +54,6 @@ trace_data/
 /src/Spark.Admin.WebApp/src/bin/
 **/CMakeDeps.cmake
 **/CMakeDeps_win.cmake
-**/.devcontainer
+.devcontainer/
 *.Generated.cs
 *.lst

CMakePresets.json

@@ -113,8 +113,8 @@
       "description": "Debug build compiled w/ gcc",
       "binaryDir": "${sourceDir}/build/gcc/debug",
       "cacheVariables": {
-        "CMAKE_C_COMPILER": "gcc",
-        "CMAKE_CXX_COMPILER": "g++",
+        "CMAKE_C_COMPILER": "gcc-12",
+        "CMAKE_CXX_COMPILER": "g++-12",
         "CMAKE_CXX_FLAGS_DEBUG": "-g -fPIC -Wall -pedantic -Wextra -Werror -Wno-missing-braces -Wno-ignored-qualifiers --std=c++20 -fdiagnostics-color -Wno-stringop-overread -DDEBUG",
         "CMAKE_INSTALL_PREFIX": "${sourceDir}/install/gcc/debug/sparkd"
       }
@@ -143,7 +143,7 @@
       "cacheVariables": {
         "CMAKE_C_COMPILER": "gcc",
         "CMAKE_CXX_COMPILER": "g++",
-        "CMAKE_CXX_FLAGS_RELEASE": "-fPIC -Wall -pedantic -Wextra -Werror -Wno-missing-braces -Wno-ignored-qualifiers --std=c++20 -fdiagnostics-color -Wno-stringop-overread -Wno-maybe-uninitialized -fno-omit-frame-pointer -DNDEBUG -O3",
+        "CMAKE_CXX_FLAGS_RELEASE": "-fPIC -Wall -pedantic -Wextra -Werror -Wno-missing-braces -Wno-ignored-qualifiers -Wno-stringop-overflow --std=c++20 -fdiagnostics-color -Wno-stringop-overread -Wno-maybe-uninitialized -fno-omit-frame-pointer -DNDEBUG -O3",
         "CMAKE_INSTALL_PREFIX": "${sourceDir}/install/gcc/release/sparkd"
       }
     },

playground-1.mongodb.js

@@ -1,45 +0,0 @@
-/* global use, db */
-// MongoDB Playground
-// To disable this template go to Settings | MongoDB | Use Default Template For Playground.
-// Make sure you are connected to enable completions and to be able to run a playground.
-// Use Ctrl+Space inside a snippet or a string literal to trigger completions.
-// The result of the last command run in a playground is shown on the results panel.
-// By default the first 20 documents will be returned with a cursor.
-// Use 'console.log()' to print to the debug output.
-// For more documentation on playgrounds please refer to
-// https://www.mongodb.com/docs/mongodb-vscode/playgrounds/
-
-// Select the database to use.
-use('mongodbVSCodePlaygroundDB');
-
-// Insert a few documents into the sales collection.
-db.getCollection('sales').insertMany([
-    { 'item': 'abc', 'price': 10, 'quantity': 2, 'date': new Date('2014-03-01T08:00:00Z') },
-    { 'item': 'jkl', 'price': 20, 'quantity': 1, 'date': new Date('2014-03-01T09:00:00Z') },
-    { 'item': 'xyz', 'price': 5, 'quantity': 10, 'date': new Date('2014-03-15T09:00:00Z') },
-    { 'item': 'xyz', 'price': 5, 'quantity': 20, 'date': new Date('2014-04-04T11:21:39.736Z') },
-    { 'item': 'abc', 'price': 10, 'quantity': 10, 'date': new Date('2014-04-04T21:23:13.331Z') },
-    { 'item': 'def', 'price': 7.5, 'quantity': 5, 'date': new Date('2015-06-04T05:08:13Z') },
-    { 'item': 'def', 'price': 7.5, 'quantity': 10, 'date': new Date('2015-09-10T08:43:00Z') },
-    { 'item': 'abc', 'price': 10, 'quantity': 5, 'date': new Date('2016-02-06T20:20:13Z') },
-]);
-
-// Run a find command to view items sold on April 4th, 2014.
-const salesOnApril4th = db.getCollection('sales').find({
-    date: { $gte: new Date('2014-04-04'), $lt: new Date('2014-04-05') }
-}).count();
-
-// Print a message to the output window.
-console.log(`${salesOnApril4th} sales occurred in 2014.`);
-
-// Here we run an aggregation and open a cursor to the results.
-// Use '.toArray()' to exhaust the cursor to return the whole result set.
-// You can use '.hasNext()/.next()' to iterate through the cursor page by page.
-db.getCollection('sales').aggregate([
-    // Find all of the sales that occurred in 2014.
-    { $match: { date: { $gte: new Date('2014-01-01'), $lt: new Date('2015-01-01') } } },
-    // Group the total sales for each product.
-    { $group: { _id: '$item', totalSaleAmount: { $sum: { $multiply: ['$price', '$quantity'] } } } }
-]);
-
-

src/DataStreamSuite/ArithmeticExpressions/ArithmeticExpressions.csproj

@@ -8,7 +8,7 @@
 
   <ItemGroup>
     <PackageReference Include="Google.Protobuf" Version="3.24.3" />
-    <PackageReference Include="YaccLexTools" Version="1.2.0" />
+    <PackageReference Include="YaccLexTools" Version="1.2.2" />
   </ItemGroup>
 
   <ItemGroup>

src/DataStreamSuite/Bts.ColumnExpressions/Bts.ColumnExpressions.csproj

@@ -0,0 +1,24 @@
+<Project Sdk="Microsoft.NET.Sdk">
+
+  <PropertyGroup>
+    <TargetFramework>net7.0</TargetFramework>
+    <ImplicitUsings>enable</ImplicitUsings>
+    <Nullable>enable</Nullable>
+  </PropertyGroup>
+
+  <ItemGroup>
+    <PackageReference Include="YaccLexTools" Version="1.2.2" />
+  </ItemGroup>
+
+  <ItemGroup>
+    <ProjectReference Include="..\DataStreamCore\DataStreamCore.csproj" />
+    <ProjectReference Include="..\ExpressionCore\ExpressionCore.csproj" />
+    <ProjectReference Include="..\..\client\spark_core\spark_core.csproj" />
+  </ItemGroup>
+
+  <ItemGroup Label="ColumnExpression">
+    <Scanner Include="ColumnExpressionTokens.lex" />
+    <Parser Include="ColumnExpressionGrammar.y" />
+  </ItemGroup>
+
+</Project>

src/DataStreamSuite/Bts.ColumnExpressions/ColumnExpression.cs

@@ -0,0 +1,62 @@
+namespace Bts.ColumnExpressions;
+
+public class ColumnExpression {
+   public ColumnExpression(string name, Type? type = null, object? value = null) {
+      Name = name;
+      Type = type
+         ?? value?.GetType()
+         ?? throw new ArgumentException(
+            "Type must be specified explicitly when value is null", nameof(type));
+      Value = value
+         ?? DBNull.Value;
+   }
+
+   public string Name { get; }
+
+   public Type Type { get; }
+
+   public object Value { get; }
+
+   public static ColumnExpression Create<T>(string name, T value, Type? type = null) {
+      // Try to convert value to 'type' to verify they are compatible
+      // Note: Can't use `System.Type.IsAssignableFrom` because it doesn't
+      // account for implicit conversions (e.g., int -> long), just inheritance.
+      if (type is not null && !ConvertUtils.try_convert(type, value, out _)) {
+         throw new ArgumentException(
+            $"Type '{type}' is not assignable from value type '{value?.GetType()}'");
+      }
+      return new(name, type ?? typeof(T), value);
+   }
+
+   internal static Type ParseType(string typeId) {
+      return Candidates().ExceptNull().FirstOrDefault() ??
+         throw new ArgumentException($"Failed to parse Type from '{typeId}'", nameof(typeId));
+
+      IEnumerable<Type?> Candidates() {
+         if (typeId.Length == 0) {
+            yield break;
+         }
+         yield return Type.GetType(typeId);
+         yield return Type.GetType($"System.{typeId}");
+         if (typeId.Length > 1) {
+            yield return Type.GetType($"System.{char.ToUpper(typeId[0]) + typeId[1..]}");
+         }
+         yield return typeId.ToLower() switch {
+            "float64" or "double" or "real64" or "number" => typeof(double),
+            "float32" or "single" or "real32" or "real" or "float" => typeof(float),
+            "int16" or "short" or "integer16" or "signed16" => typeof(short),
+            "int32" or "int" or "integer" or "integer32" or "signed" => typeof(int),
+            "int64" or "long" or "integer64" or "signed64" => typeof(long),
+            "uint16" or "ushort" or "unsigned16" => typeof(ushort),
+            "uint32" or "uint" or "unsigned" => typeof(uint),
+            "uint64" or "ulong" or "unsigned64" => typeof(ulong),
+            "char" => typeof(char),
+            "byte" or "uint8" => typeof(byte),
+            "bool" or "boolean" => typeof(bool),
+            "string" or "text" => typeof(string),
+            "date" or "datetime" => typeof(DateTime),
+            _ => null
+         };
+      }
+   }
+}

src/DataStreamSuite/Bts.ColumnExpressions/ColumnExpressionGrammar.y

@@ -0,0 +1,49 @@
+%namespace Bts.ColumnExpressions
+%partial
+%visibility internal
+%tokentype ColumnExpressionToken
+%parsertype ColumnExpressionParser
+
+%union {
+   public int Integer;
+   public double Real;
+   public bool Boolean;
+   public string String;
+   public string Identifier;
+   public string TypeId;
+   public DateTime DateTime;
+   public Tuple<string, Type> ColumnDefinition;
+   public ColumnExpression ColumnExpression;
+   public ImmutableList<ColumnExpression> ColumnExpressions;
+}
+
+%token BOOLEAN_CONSTANT, INTEGER_CONSTANT, REAL_CONSTANT, DATE_TIME_CONSTANT, STRING_CONSTANT
+%token IDENTIFIER, EQUALS, TYPE_ID, OF_TYPE
+
+%nonassoc EQUALS
+
+%type <ColumnDefinition> ColumnDefinition
+%type <ColumnExpression> ColumnExpression
+%type <ColumnExpressions> ColumnExpressions
+
+%%
+
+ColumnExpressions :
+   ColumnExpression { $$ = ImmutableList.CreateRange(new[] { $1 }); }
+   | ColumnExpressions ColumnExpression { $$ = $1.Add($2); }
+   ;
+
+ColumnExpression :
+   ColumnDefinition EQUALS INTEGER_CONSTANT { $$ = ColumnExpression.Create($1.Item1, $3.Integer, $1.Item2); }
+   | ColumnDefinition EQUALS REAL_CONSTANT { $$ = ColumnExpression.Create($1.Item1, $3.Real, $1.Item2); }
+   | ColumnDefinition EQUALS DATE_TIME_CONSTANT { $$ = ColumnExpression.Create($1.Item1, $3.DateTime, $1.Item2); }
+   | ColumnDefinition EQUALS BOOLEAN_CONSTANT { $$ = ColumnExpression.Create($1.Item1, $3.Boolean, $1.Item2); }
+   | ColumnDefinition EQUALS STRING_CONSTANT { $$ = ColumnExpression.Create($1.Item1, $3.String, $1.Item2); }
+   | ColumnDefinition { $$ = new($1.Item1, $1.Item2); }
+   ;
+
+ColumnDefinition :
+   IDENTIFIER { $$ = new($1.Identifier, null); }
+   | IDENTIFIER OF_TYPE IDENTIFIER { $$ = new($1.Identifier, ColumnExpression.ParseType($3.Identifier)); }
+   | IDENTIFIER OF_TYPE TYPE_ID { $$ = new($1.Identifier, ColumnExpression.ParseType($3.TypeId)); }
+   ; 

src/DataStreamSuite/Bts.ColumnExpressions/ColumnExpressionInterpreter.cs

@@ -0,0 +1,18 @@
+namespace Bts.ColumnExpressions;
+
+using System.IO;
+using System.Text;
+
+public class ColumnExpressionInterpreter {
+   public static IList<ColumnExpression> Evaluate(Stream stream) {
+      var parser = new ColumnExpressionParser();
+      var res = parser.Parse(stream);
+      return res?.ColumnExpressions ?? ImmutableList<ColumnExpression>.Empty.Add(
+         res?.ColumnExpression ?? throw new Exception("Failed to parse column(s) expression"));
+   }
+
+   public static IList<ColumnExpression> Evaluate(string str) {
+      using var ms = new MemoryStream(Encoding.Default.GetBytes(str));
+      return Evaluate(ms);
+   }
+}

src/DataStreamSuite/Bts.ColumnExpressions/ColumnExpressionParser.cs

@@ -0,0 +1,18 @@
+namespace Bts.ColumnExpressions;
+
+using System.Text;
+
+partial class ColumnExpressionParser {
+   public ColumnExpressionParser() : base(null) { }
+
+   public ValueType? Parse(string s) {
+      using var ms = new MemoryStream(Encoding.Default.GetBytes(s));
+      return Parse(ms);
+   }
+
+   public ValueType? Parse(Stream stream) {
+      Scanner = new ColumnExpressionScanner(stream);
+      Parse();
+      return CurrentSemanticValue;
+   }
+}

src/DataStreamSuite/Bts.ColumnExpressions/ColumnExpressionScanner.cs

@@ -0,0 +1,9 @@
+namespace Bts.ColumnExpressions;
+
+partial class ColumnExpressionScanner {
+   public override void yyerror(string format, params object[] args) {
+      base.yyerror(format, args);
+      Console.WriteLine(format, args);
+      Console.WriteLine();
+   }
+}

src/DataStreamSuite/Bts.ColumnExpressions/ColumnExpressionTokens.lex

@@ -0,0 +1,85 @@
+%namespace Bts.ColumnExpressions
+%scannertype ColumnExpressionScanner
+%visibility internal
+%tokentype ColumnExpressionToken
+
+%option stack, minimize, parser, verbose, persistbuffer, noembedbuffers 
+
+/* Literal Constants */
+Letter                   [a-zA-Z]
+BooleanConstant          true|false
+StringConstant           '([^\'\\]*|\\.)*'
+DateTimeConstant         ([1-9]|'10'|'11'|'12')\/([1-9]|([1-2][0-9])|'30'|'31')\/[0-9][0-9][0-9]*
+IntegerConstant          -?[0-9]+
+NullConstant             'null'
+RealConstant             -?[0-9]+\.[0-9]+
+
+/* Relational Operators */
+Equals                   =
+OfType                   /
+
+Identifier               {Letter}({Letter}*[0-9]*_*)*
+NamespacePrefix          {Identifier}\.
+TypeId                   {NamespacePrefix}*{Identifier}
+
+/* Utility Tokens */
+OpenParenthesis          \(
+CloseParenthesis         \)
+Comma                    ,
+QuoteOpen                \[
+QuoteClose               \]
+
+/* Comments */
+LineComment \/\/[^\n\r]*(\n|\r|\r\n)
+
+%%
+
+{Equals} {
+   return (int)ColumnExpressionToken.EQUALS;
+}
+
+{OfType} {
+   return (int)ColumnExpressionToken.OF_TYPE;
+}
+
+{StringConstant} {
+   yylval.String = yytext.Substring(1, yytext.Length - 2).Replace(@"\'","'");
+   return (int)ColumnExpressionToken.STRING_CONSTANT;
+}
+
+{BooleanConstant} {
+   yylval.Boolean = yytext.ToLower() switch {
+      "true" => true,
+      "false" => false,
+      _ => throw new Exception(
+         string.Format("Expected yytext to be 'true' or 'false' (actually '{0}')", yytext))
+   };
+   return (int)ColumnExpressionToken.BOOLEAN_CONSTANT;
+}
+
+{DateTimeConstant} {
+   yylval.DateTime = DateTime.Parse(yytext);
+   return (int)ColumnExpressionToken.DATE_TIME_CONSTANT;
+}
+
+{IntegerConstant} {
+   yylval.Integer = int.Parse(yytext);
+   return (int)ColumnExpressionToken.INTEGER_CONSTANT;
+}
+
+{RealConstant} {
+   yylval.Real = double.Parse(yytext);
+   return (int)ColumnExpressionToken.REAL_CONSTANT;
+}
+
+{Identifier} {
+   yylval.Identifier = yytext;
+   return (int)ColumnExpressionToken.IDENTIFIER;
+}
+
+{TypeId} {
+   yylval.TypeId = yytext;   
+   return (int)ColumnExpressionToken.TYPE_ID;
+}
+
+%%

src/DataStreamSuite/Bts.ColumnExpressions/GplexBuffers.cs

@@ -0,0 +1,720 @@
+// ==============================================================
+// <auto-generated>
+// This code automatically produced from an embedded resource.
+// Do not edit this file, or it will become incompatible with 
+// the specification from which it was generated.
+// </auto-generated>
+// ==============================================================
+
+using System;
+using System.IO;
+using System.Text;
+using System.Collections.Generic;
+using System.Diagnostics.CodeAnalysis;
+using System.Runtime.Serialization;
+using System.Globalization;
+
+namespace QUT.GplexBuffers
+{
+// Code copied from GPLEX embedded resource
+    [Serializable]
+    internal class BufferException : Exception
+    {
+        public BufferException() { }
+        public BufferException(string message) : base(message) { }
+        public BufferException(string message, Exception innerException)
+            : base(message, innerException) { }
+        protected BufferException(SerializationInfo info, StreamingContext context)
+            : base(info, context) { }
+    }
+
+    internal abstract class ScanBuff
+    {
+        private string fileNm;
+
+        public const int EndOfFile = -1;
+        public const int UnicodeReplacementChar = 0xFFFD;
+
+        public bool IsFile { get { return (fileNm != null); } }
+        public string FileName { get { return fileNm; } set { fileNm = value; } }
+
+        public abstract int Pos { get; set; }
+        public abstract int Read();
+        public virtual void Mark() { }
+
+        public abstract string GetString(int begin, int limit);
+
+        public static ScanBuff GetBuffer(string source)
+        {
+            return new StringBuffer(source);
+        }
+
+        public static ScanBuff GetBuffer(IList<string> source)
+        {
+            return new LineBuffer(source);
+        }
+
+#if (!NOFILES)
+        public static ScanBuff GetBuffer(Stream source)
+        {
+            return new BuildBuffer(source);
+        }
+
+#if (!BYTEMODE)
+        public static ScanBuff GetBuffer(Stream source, int fallbackCodePage)
+        {
+            return new BuildBuffer(source, fallbackCodePage);
+        }
+#endif // !BYTEMODE
+#endif // !NOFILES
+    }
+
+    #region Buffer classes
+
+    // ==============================================================
+    // =====  Definitions for various ScanBuff derived classes   ====
+    // ==============================================================
+    // ===============         String input          ================
+    // ==============================================================
+
+    /// <summary>
+    /// This class reads characters from a single string as
+    /// required, for example, by Visual Studio language services
+    /// </summary>
+    sealed class StringBuffer : ScanBuff
+    {
+        string str;        // input buffer
+        int bPos;          // current position in buffer
+        int sLen;
+
+        public StringBuffer(string source)
+        {
+            this.str = source;
+            this.sLen = source.Length;
+            this.FileName = null;
+        }
+
+        public override int Read()
+        {
+            if (bPos < sLen) return str[bPos++];
+            else if (bPos == sLen) { bPos++; return '\n'; }   // one strike, see new line
+            else { bPos++; return EndOfFile; }                // two strikes and you're out!
+        }
+
+        public override string GetString(int begin, int limit)
+        {
+            //  "limit" can be greater than sLen with the BABEL
+            //  option set.  Read returns a "virtual" EOL if
+            //  an attempt is made to read past the end of the
+            //  string buffer.  Without the guard any attempt 
+            //  to fetch yytext for a token that includes the 
+            //  EOL will throw an index exception.
+            if (limit > sLen) limit = sLen;
+            if (limit <= begin) return "";
+            else return str.Substring(begin, limit - begin);
+        }
+
+        public override int Pos
+        {
+            get { return bPos; }
+            set { bPos = value; }
+        }
+
+        public override string ToString() { return "StringBuffer"; }
+    }
+
+    // ==============================================================
+    //  The LineBuff class contributed by Nigel Horspool, 
+    //  nigelh@cs.uvic.cs
+    // ==============================================================
+
+    sealed class LineBuffer : ScanBuff
+    {
+        IList<string> line;    // list of source lines from a file
+        int numLines;          // number of strings in line list
+        string curLine;        // current line in that list
+        int cLine;             // index of current line in the list
+        int curLen;            // length of current line
+        int curLineStart;      // position of line start in whole file
+        int curLineEnd;        // position of line end in whole file
+        int maxPos;            // max position ever visited in whole file
+        int cPos;              // ordinal number of code in source
+
+        // Constructed from a list of strings, one per source line.
+        // The lines have had trailing '\n' characters removed.
+        public LineBuffer(IList<string> lineList)
+        {
+            line = lineList;
+            numLines = line.Count;
+            cPos = curLineStart = 0;
+            curLine = (numLines > 0 ? line[0] : "");
+            maxPos = curLineEnd = curLen = curLine.Length;
+            cLine = 1;
+            FileName = null;
+        }
+
+        public override int Read()
+        {
+            if (cPos < curLineEnd)
+                return curLine[cPos++ - curLineStart];
+            if (cPos++ == curLineEnd)
+                return '\n';
+            if (cLine >= numLines)
+                return EndOfFile;
+            curLine = line[cLine];
+            curLen = curLine.Length;
+            curLineStart = curLineEnd + 1;
+            curLineEnd = curLineStart + curLen;
+            if (curLineEnd > maxPos)
+                maxPos = curLineEnd;
+            cLine++;
+            return curLen > 0 ? curLine[0] : '\n';
+        }
+
+        // To speed up searches for the line containing a position
+        private int cachedPosition;
+        private int cachedIxdex;
+        private int cachedLineStart;
+
+        // Given a position pos within the entire source, the results are
+        //   ix     -- the index of the containing line
+        //   lstart -- the position of the first character on that line
+        private void findIndex(int pos, out int ix, out int lstart)
+        {
+            if (pos >= cachedPosition)
+            {
+                ix = cachedIxdex; lstart = cachedLineStart;
+            }
+            else
+            {
+                ix = lstart = 0;
+            }
+            while (ix < numLines)
+            {
+                int len = line[ix].Length + 1;
+                if (pos < lstart + len) break;
+                lstart += len;
+                ix++;
+            }
+            cachedPosition = pos;
+            cachedIxdex = ix;
+            cachedLineStart = lstart;
+        }
+
+        public override string GetString(int begin, int limit)
+        {
+            if (begin >= maxPos || limit <= begin) return "";
+            int endIx, begIx, endLineStart, begLineStart;
+            findIndex(begin, out begIx, out begLineStart);
+            int begCol = begin - begLineStart;
+            findIndex(limit, out endIx, out endLineStart);
+            int endCol = limit - endLineStart;
+            string s = line[begIx];
+            if (begIx == endIx)
+            {
+                // the usual case, substring all on one line
+                return (endCol <= s.Length) ?
+                    s.Substring(begCol, endCol - begCol)
+                    : s.Substring(begCol) + "\n";
+            }
+            // the string spans multiple lines, yuk!
+            StringBuilder sb = new StringBuilder();
+            if (begCol < s.Length)
+                sb.Append(s.Substring(begCol));
+            for (; ; )
+            {
+                sb.Append("\n");
+                s = line[++begIx];
+                if (begIx >= endIx) break;
+                sb.Append(s);
+            }
+            if (endCol <= s.Length)
+            {
+                sb.Append(s.Substring(0, endCol));
+            }
+            else
+            {
+                sb.Append(s);
+                sb.Append("\n");
+            }
+            return sb.ToString();
+        }
+
+        public override int Pos
+        {
+            get { return cPos; }
+            set
+            {
+                cPos = value;
+                findIndex(cPos, out cLine, out curLineStart);
+                // cLine should be the *next* line after curLine.
+                curLine = (cLine < numLines ? line[cLine++] : "");
+                curLineEnd = curLineStart + curLine.Length;
+            }
+        }
+
+        public override string ToString() { return "LineBuffer"; }
+    }
+
+#if (!NOFILES)
+    // ==============================================================
+    // =====     class BuildBuff : for unicode text files    ========
+    // ==============================================================
+
+    class BuildBuffer : ScanBuff
+    {
+        // Double buffer for char stream.
+        class BufferElement
+        {
+            StringBuilder bldr = new StringBuilder();
+            StringBuilder next = new StringBuilder();
+            int minIx;
+            int maxIx;
+            int brkIx;
+            bool appendToNext;
+
+            internal BufferElement() { }
+
+            internal int MaxIndex { get { return maxIx; } }
+            // internal int MinIndex { get { return minIx; } }
+
+            internal char this[int index]
+            {
+                get
+                {
+                    if (index < minIx || index >= maxIx)
+                        throw new BufferException("Index was outside data buffer");
+                    else if (index < brkIx)
+                        return bldr[index - minIx];
+                    else
+                        return next[index - brkIx];
+                }
+            }
+
+            internal void Append(char[] block, int count)
+            {
+                maxIx += count;
+                if (appendToNext)
+                    this.next.Append(block, 0, count);
+                else
+                {
+                    this.bldr.Append(block, 0, count);
+                    brkIx = maxIx;
+                    appendToNext = true;
+                }
+            }
+
+            internal string GetString(int start, int limit)
+            {
+                if (limit <= start)
+                    return "";
+                if (start >= minIx && limit <= maxIx)
+                    if (limit < brkIx) // String entirely in bldr builder
+                        return bldr.ToString(start - minIx, limit - start);
+                    else if (start >= brkIx) // String entirely in next builder
+                        return next.ToString(start - brkIx, limit - start);
+                    else // Must do a string-concatenation
+                        return
+                            bldr.ToString(start - minIx, brkIx - start) +
+                            next.ToString(0, limit - brkIx);
+                else
+                    throw new BufferException("String was outside data buffer");
+            }
+
+            internal void Mark(int limit)
+            {
+                if (limit > brkIx + 16) // Rotate blocks
+                {
+                    StringBuilder temp = bldr;
+                    bldr = next;
+                    next = temp;
+                    next.Length = 0;
+                    minIx = brkIx;
+                    brkIx = maxIx;
+                }
+            }
+        }
+
+        BufferElement data = new BufferElement();
+
+        int bPos;            // Postion index in the StringBuilder
+        BlockReader NextBlk; // Delegate that serves char-arrays;
+
+        private string EncodingName
+        {
+            get
+            {
+                StreamReader rdr = NextBlk.Target as StreamReader;
+                return (rdr == null ? "raw-bytes" : rdr.CurrentEncoding.BodyName);
+            }
+        }
+
+        public BuildBuffer(Stream stream)
+        {
+            FileStream fStrm = (stream as FileStream);
+            if (fStrm != null) FileName = fStrm.Name;
+            NextBlk = BlockReaderFactory.Raw(stream);
+        }
+
+#if (!BYTEMODE)
+        public BuildBuffer(Stream stream, int fallbackCodePage)
+        {
+            FileStream fStrm = (stream as FileStream);
+            if (fStrm != null) FileName = fStrm.Name;
+            NextBlk = BlockReaderFactory.Get(stream, fallbackCodePage);
+        }
+#endif
+
+        /// <summary>
+        /// Marks a conservative lower bound for the buffer,
+        /// allowing space to be reclaimed.  If an application 
+        /// needs to call GetString at arbitrary past locations 
+        /// in the input stream, Mark() is not called.
+        /// </summary>
+        public override void Mark() { data.Mark(bPos - 2); }
+
+        public override int Pos
+        {
+            get { return bPos; }
+            set { bPos = value; }
+        }
+
+
+        /// <summary>
+        /// Read returns the ordinal number of the next char, or 
+        /// EOF (-1) for an end of stream.  Note that the next
+        /// code point may require *two* calls of Read().
+        /// </summary>
+        /// <returns></returns>
+        public override int Read()
+        {
+            //
+            //  Characters at positions 
+            //  [data.offset, data.offset + data.bldr.Length)
+            //  are available in data.bldr.
+            //
+            if (bPos < data.MaxIndex)
+            {
+                // ch0 cannot be EOF
+                return (int)data[bPos++];
+            }
+            else // Read from underlying stream
+            {
+                // Experimental code, blocks of page size
+                char[] chrs = new char[4096];
+                int count = NextBlk(chrs, 0, 4096);
+                if (count == 0)
+                    return EndOfFile;
+                else
+                {
+                    data.Append(chrs, count);
+                    return (int)data[bPos++];
+                }
+            }
+        }
+
+        public override string GetString(int begin, int limit)
+        {
+            return data.GetString(begin, limit);
+        }
+
+        public override string ToString()
+        {
+            return "StringBuilder buffer, encoding: " + this.EncodingName;
+        }
+    }
+
+    // =============== End ScanBuff-derived classes ==================
+
+    internal delegate int BlockReader(char[] block, int index, int number);
+
+    // A delegate factory, serving up a delegate that
+    // reads a block of characters from the underlying
+    // encoded stream, via a StreamReader object.
+    //
+    internal static class BlockReaderFactory
+    {
+        public static BlockReader Raw(Stream stream)
+        {
+            return delegate(char[] block, int index, int number)
+            {
+                byte[] b = new byte[number];
+                int count = stream.Read(b, 0, number);
+                int i = 0;
+                int j = index;
+                for (; i < count; i++, j++)
+                    block[j] = (char)b[i];
+                return count;
+            };
+        }
+
+#if (!BYTEMODE)
+        public static BlockReader Get(Stream stream, int fallbackCodePage)
+        {
+            Encoding encoding;
+            int preamble = Preamble(stream);
+
+            if (preamble != 0)  // There is a valid BOM here!
+                encoding = Encoding.GetEncoding(preamble);
+            else if (fallbackCodePage == -1) // Fallback is "raw" bytes
+                return Raw(stream);
+            else if (fallbackCodePage != -2) // Anything but "guess"
+                encoding = Encoding.GetEncoding(fallbackCodePage);
+            else // This is the "guess" option
+            {
+                int guess = new Guesser(stream).GuessCodePage();
+                stream.Seek(0, SeekOrigin.Begin);
+                if (guess == -1) // ==> this is a 7-bit file
+                    encoding = Encoding.ASCII;
+                else if (guess == 65001)
+                    encoding = Encoding.UTF8;
+                else             // ==> use the machine default
+                    encoding = Encoding.Default;
+            }
+            StreamReader reader = new StreamReader(stream, encoding);
+            return reader.Read;
+        }
+
+        static int Preamble(Stream stream)
+        {
+            int b0 = stream.ReadByte();
+            int b1 = stream.ReadByte();
+
+            if (b0 == 0xfe && b1 == 0xff)
+                return 1201; // UTF16BE
+            if (b0 == 0xff && b1 == 0xfe)
+                return 1200; // UTF16LE
+
+            int b2 = stream.ReadByte();
+            if (b0 == 0xef && b1 == 0xbb && b2 == 0xbf)
+                return 65001; // UTF8
+            //
+            // There is no unicode preamble, so we
+            // return denoter for the machine default.
+            //
+            stream.Seek(0, SeekOrigin.Begin);
+            return 0;
+        }
+#endif // !BYTEMODE
+    }
+#endif // !NOFILES
+    #endregion Buffer classes
+
+    // ==============================================================
+    // ============      class CodePageHandling         =============
+    // ==============================================================
+#if (!NOFILES)
+    public static class CodePageHandling
+    {
+        public static int GetCodePage(string option)
+        {
+            string command = option.ToUpperInvariant();
+            if (command.StartsWith("CodePage:", StringComparison.OrdinalIgnoreCase))
+                command = command.Substring(9);
+            try
+            {
+                if (command.Equals("RAW"))
+                    return -1;
+                else if (command.Equals("GUESS"))
+                    return -2;
+                else if (command.Equals("DEFAULT"))
+                    return 0;
+                else if (char.IsDigit(command[0]))
+                    return int.Parse(command, CultureInfo.InvariantCulture);
+                else
+                {
+                    Encoding enc = Encoding.GetEncoding(command);
+                    return enc.CodePage;
+                }
+            }
+            catch (FormatException)
+            {
+                Console.Error.WriteLine(
+                    "Invalid format \"{0}\", using machine default", option);
+            }
+            catch (ArgumentException)
+            {
+                Console.Error.WriteLine(
+                    "Unknown code page \"{0}\", using machine default", option);
+            }
+            return 0;
+        }
+    }
+#region guesser
+#if (!BYTEMODE)
+    // ==============================================================
+    // ============          Encoding Guesser           =============
+    // ==============================================================
+
+    /// <summary>
+    /// This class provides a simple finite state automaton that
+    /// scans the file looking for (1) valid UTF-8 byte patterns,
+    /// (2) bytes >= 0x80 which are not part of a UTF-8 sequence.
+    /// The method then guesses whether it is UTF-8 or maybe some 
+    /// local machine default encoding.  This works well for the
+    /// various Latin encodings.
+    /// </summary>
+    internal class Guesser
+    {
+        ScanBuff buffer;
+
+        public int GuessCodePage() { return Scan(); }
+
+        const int maxAccept = 10;
+        const int initial = 0;
+        const int eofNum = 0;
+        const int goStart = -1;
+        const int INITIAL = 0;
+        const int EndToken = 0;
+
+        #region user code
+        /* 
+         *  Reads the bytes of a file to determine if it is 
+         *  UTF-8 or a single-byte code page file.
+         */
+        public long utfX;
+        public long uppr;
+        #endregion user code
+
+        int state;
+        int currentStart = startState[0];
+        int code;
+
+        #region ScannerTables
+        static int[] startState = new int[] { 11, 0 };
+
+        #region CharacterMap
+        static sbyte[] map = new sbyte[256] {
+/*     '\0' */ 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 
+/*   '\x10' */ 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 
+/*   '\x20' */ 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 
+/*      '0' */ 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 
+/*      '@' */ 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 
+/*      'P' */ 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 
+/*      '`' */ 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 
+/*      'p' */ 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 
+/*   '\x80' */ 2, 2, 2, 2, 2, 2, 2, 2, 2, 2, 2, 2, 2, 2, 2, 2, 
+/*   '\x90' */ 2, 2, 2, 2, 2, 2, 2, 2, 2, 2, 2, 2, 2, 2, 2, 2, 
+/*   '\xA0' */ 2, 2, 2, 2, 2, 2, 2, 2, 2, 2, 2, 2, 2, 2, 2, 2, 
+/*   '\xB0' */ 2, 2, 2, 2, 2, 2, 2, 2, 2, 2, 2, 2, 2, 2, 2, 2, 
+/*   '\xC0' */ 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 
+/*   '\xD0' */ 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 
+/*   '\xE0' */ 3, 3, 3, 3, 3, 3, 3, 3, 3, 3, 3, 3, 3, 3, 3, 3, 
+/*   '\xF0' */ 4, 4, 4, 4, 4, 4, 4, 4, 5, 5, 5, 5, 5, 5, 5, 5 };
+        #endregion
+
+        static sbyte[][] nextState = new sbyte[][] {
+            new sbyte[] {0, 0, 0, 0, 0, 0},
+            new sbyte[] {-1, -1, 10, -1, -1, -1},
+            new sbyte[] {-1, -1, -1, -1, -1, -1},
+            new sbyte[] {-1, -1, 8, -1, -1, -1},
+            new sbyte[] {-1, -1, 5, -1, -1, -1},
+            new sbyte[] {-1, -1, 6, -1, -1, -1},
+            new sbyte[] {-1, -1, 7, -1, -1, -1},
+            null,
+            new sbyte[] {-1, -1, 9, -1, -1, -1},
+            null,
+            null,
+            new sbyte[] {-1, 1, 2, 3, 4, 2}
+        };
+
+
+        [SuppressMessage("Microsoft.Performance", "CA1810:InitializeReferenceTypeStaticFieldsInline")]
+        // Reason for suppression: cannot have self-reference in array initializer.
+        static Guesser()
+        {
+            nextState[7] = nextState[2];
+            nextState[9] = nextState[2];
+            nextState[10] = nextState[2];
+        }
+
+        int NextState()
+        {
+            if (code == ScanBuff.EndOfFile)
+                return eofNum;
+            else
+                return nextState[state][map[code]];
+        }
+        #endregion
+
+        public Guesser(System.IO.Stream file) { SetSource(file); }
+
+        public void SetSource(System.IO.Stream source)
+        {
+            this.buffer = new BuildBuffer(source);
+            code = buffer.Read();
+        }
+
+        int Scan()
+        {
+            for (; ; )
+            {
+                int next;
+                state = currentStart;
+                while ((next = NextState()) == goStart)
+                    code = buffer.Read();
+
+                state = next;
+                code = buffer.Read();
+
+                while ((next = NextState()) > eofNum)
+                {
+                    state = next;
+                    code = buffer.Read();
+                }
+                if (state <= maxAccept)
+                {
+                    #region ActionSwitch
+#pragma warning disable 162
+                    switch (state)
+                    {
+                        case eofNum:
+                            switch (currentStart)
+                            {
+                                case 11:
+                                    if (utfX == 0 && uppr == 0) return -1; /* raw ascii */
+                                    else if (uppr * 10 > utfX) return 0;   /* default code page */
+                                    else return 65001;                     /* UTF-8 encoding */
+                                    break;
+                            }
+                            return EndToken;
+                        case 1: // Recognized '{Upper128}',	Shortest string "\xC0"
+                        case 2: // Recognized '{Upper128}',	Shortest string "\x80"
+                        case 3: // Recognized '{Upper128}',	Shortest string "\xE0"
+                        case 4: // Recognized '{Upper128}',	Shortest string "\xF0"
+                            uppr++;
+                            break;
+                        case 5: // Recognized '{Utf8pfx4}{Utf8cont}',	Shortest string "\xF0\x80"
+                            uppr += 2;
+                            break;
+                        case 6: // Recognized '{Utf8pfx4}{Utf8cont}{2}',	Shortest string "\xF0\x80\x80"
+                            uppr += 3;
+                            break;
+                        case 7: // Recognized '{Utf8pfx4}{Utf8cont}{3}',	Shortest string "\xF0\x80\x80\x80"
+                            utfX += 3;
+                            break;
+                        case 8: // Recognized '{Utf8pfx3}{Utf8cont}',	Shortest string "\xE0\x80"
+                            uppr += 2;
+                            break;
+                        case 9: // Recognized '{Utf8pfx3}{Utf8cont}{2}',	Shortest string "\xE0\x80\x80"
+                            utfX += 2;
+                            break;
+                        case 10: // Recognized '{Utf8pfx2}{Utf8cont}',	Shortest string "\xC0\x80"
+                            utfX++;
+                            break;
+                        default:
+                            break;
+                    }
+#pragma warning restore 162
+                    #endregion
+                }
+            }
+        }
+    } // end class Guesser
+    
+#endif // !BYTEMODE
+#endregion
+#endif // !NOFILES
+
+// End of code copied from embedded resource
+}

src/DataStreamSuite/Bts.ColumnExpressions/ShiftReduceParserCode.cs

@@ -0,0 +1,884 @@
+// ==============================================================
+// <auto-generated>
+// This code automatically produced from an embedded resource.
+// Do not edit this file, or it will become incompatible with 
+// the specification from which it was generated.
+// </auto-generated>
+// ==============================================================
+
+// Gardens Point Parser Generator
+// Copyright (c) Wayne Kelly, QUT 2005-2010
+// (see accompanying GPPGcopyright.rtf)
+
+using System;
+using System.Text;
+using System.Globalization;
+using System.Collections.Generic;
+using System.Runtime.Serialization;
+using System.Diagnostics.CodeAnalysis;
+
+namespace QUT.Gppg {
+    /// <summary>
+    /// Abstract class for GPPG shift-reduce parsers.
+    /// Parsers generated by GPPG derive from this base
+    /// class, overriding the abstract Initialize() and
+    /// DoAction() methods.
+    /// </summary>
+    /// <typeparam name="TValue">Semantic value type</typeparam>
+    /// <typeparam name="TSpan">Location type</typeparam>
+#if EXPORT_GPPG
+    public abstract class ShiftReduceParser<TValue, TSpan>
+#else
+    internal abstract class ShiftReduceParser<TValue, TSpan>
+#endif
+ where TSpan : IMerge<TSpan>, new() {
+        private AbstractScanner<TValue, TSpan> scanner;
+        /// <summary>
+        /// The abstract scanner for this parser.
+        /// </summary>
+        protected AbstractScanner<TValue, TSpan> Scanner {
+            get { return scanner; }
+            set { scanner = value; }
+        }
+
+        /// <summary>
+        /// Constructor for base class
+        /// </summary>
+        /// <param name="scanner">Scanner instance for this parser</param>
+        protected ShiftReduceParser( AbstractScanner<TValue, TSpan> scanner ) {
+            this.scanner = scanner;
+        }
+
+        // ==============================================================
+        //                    TECHNICAL EXPLANATION.
+        //   Why the next two fields are not exposed via properties.
+        // ==============================================================
+        // These fields are of the generic parameter types, and are
+        // frequently instantiated as struct types in derived classes.
+        // Semantic actions are defined in the derived classes and refer
+        // to instance fields of these structs.  In such cases the code
+        // "get_CurrentSemanticValue().myField = blah;" will fail since
+        // the getter pushes the value of the field, not the reference.
+        // So, in the presence of properties, gppg would need to encode
+        // such field accesses as ... 
+        //  "tmp = get_CurrentSemanticValue(); // Fetch value
+        //   tmp.myField = blah;               // update
+        //   set_CurrentSemanticValue(tmp); "  // Write update back.
+        // There is no issue if TValue is restricted to be a ref type.
+        // The same explanation applies to scanner.yylval.
+        // ==============================================================
+        /// <summary>
+        /// The current value of the "$$" symbolic variable in the parser
+        /// </summary>
+        [SuppressMessage( "Microsoft.Design", "CA1051:DoNotDeclareVisibleInstanceFields" )]
+        protected TValue CurrentSemanticValue;
+
+        /// <summary>
+        /// The current value of the "@$" symbolic variable in the parser
+        /// </summary>
+        [SuppressMessage( "Microsoft.Design", "CA1051:DoNotDeclareVisibleInstanceFields" )]
+        protected TSpan CurrentLocationSpan;
+        protected int NextToken;
+
+        private TSpan LastSpan;
+        private State FsaState;
+        private bool recovering;
+        private int tokensSinceLastError;
+
+        private PushdownPrefixState<State> StateStack = new PushdownPrefixState<State>();
+        private PushdownPrefixState<TValue> valueStack = new PushdownPrefixState<TValue>();
+        private PushdownPrefixState<TSpan> locationStack = new PushdownPrefixState<TSpan>();
+
+        /// <summary>
+        /// The stack of semantic value (YYSTYPE) values.
+        /// </summary>
+        protected PushdownPrefixState<TValue> ValueStack { get { return valueStack; } }
+
+        /// <summary>
+        /// The stack of location value (YYLTYPE) varlues.
+        /// </summary>
+        protected PushdownPrefixState<TSpan> LocationStack { get { return locationStack; } }
+
+        private int errorToken;
+        private int endOfFileToken;
+        private string[] nonTerminals;
+        private State[] states;
+        private Rule[] rules;
+
+        /// <summary>
+        /// Initialization method to allow derived classes
+        /// to insert the rule list into this base class.
+        /// </summary>
+        /// <param name="rules">The array of Rule objects</param>
+        protected void InitRules( Rule[] rules ) { this.rules = rules; }
+
+        /// <summary>
+        /// Initialization method to allow derived classes to
+        /// insert the states table into this base class.
+        /// </summary>
+        /// <param name="states">The pre-initialized states table</param>
+        protected void InitStates( State[] states ) { this.states = states; }
+
+        /// <summary>
+        /// OBSOLETE FOR VERSION 1.4.0
+        /// </summary>
+        /// <param name="size"></param>
+        protected void InitStateTable( int size ) { states = new State[size]; }
+
+        /// <summary>
+        /// Initialization method to allow derived classes
+        /// to insert the special value for the error and EOF tokens.
+        /// </summary>
+        /// <param name="err">The error state ordinal</param>
+        /// <param name="end">The EOF stat ordinal</param>
+        protected void InitSpecialTokens( int err, int end ) {
+            errorToken = err;
+            endOfFileToken = end;
+        }
+
+        /// <summary>
+        /// Initialization method to allow derived classes to
+        /// insert the non-terminal symbol names into this base class.
+        /// </summary>
+        /// <param name="names">Non-terminal symbol names</param>
+        protected void InitNonTerminals( string[] names ) { nonTerminals = names; }
+
+        #region YYAbort, YYAccept etcetera.
+        [Serializable]
+        [SuppressMessage( "Microsoft.Design", "CA1064:ExceptionsShouldBePublic" )]
+        // Reason for FxCop message suppression -
+        // This exception cannot escape from the local context
+        private class AcceptException : Exception {
+            internal AcceptException() { }
+            protected AcceptException( SerializationInfo i, StreamingContext c ) : base( i, c ) { }
+        }
+        [Serializable]
+        [SuppressMessage( "Microsoft.Design", "CA1064:ExceptionsShouldBePublic" )]
+        // Reason for FxCop message suppression -
+        // This exception cannot escape from the local context
+        private class AbortException : Exception {
+            internal AbortException() { }
+            protected AbortException( SerializationInfo i, StreamingContext c ) : base( i, c ) { }
+        }
+        [Serializable]
+        [SuppressMessage( "Microsoft.Design", "CA1064:ExceptionsShouldBePublic" )]
+        // Reason for FxCop message suppression -
+        // This exception cannot escape from the local context
+        private class ErrorException : Exception {
+            internal ErrorException() { }
+            protected ErrorException( SerializationInfo i, StreamingContext c ) : base( i, c ) { }
+        }
+
+        // The following methods are only called from within
+        // a semantic action. The thrown exceptions can never
+        // propagate outside the ShiftReduceParser class in 
+        // which they are nested.
+
+        /// <summary>
+        /// Force parser to terminate, returning "true"
+        /// </summary>
+        protected static void YYAccept() { throw new AcceptException(); }
+
+        /// <summary>
+        /// Force parser to terminate, returning "false"
+        /// </summary>
+        protected static void YYAbort() { throw new AbortException(); }
+
+        /// <summary>
+        /// Force parser to terminate, returning
+        /// "false" if error recovery fails.
+        /// </summary>
+        protected static void YYError() { throw new ErrorException(); }
+
+        /// <summary>
+        /// Check if parser in error recovery state.
+        /// </summary>
+        protected bool YYRecovering { get { return recovering; } }
+        #endregion
+
+        /// <summary>
+        /// Abstract base method. ShiftReduceParser calls this
+        /// to initialize the base class data structures.  Concrete
+        /// parser classes must override this method.
+        /// </summary>
+        protected abstract void Initialize();
+
+        /// <summary>
+        /// Main entry point of the Shift-Reduce Parser.
+        /// </summary>
+        /// <returns>True if parse succeeds, else false for
+        /// unrecoverable errors</returns>
+        public bool Parse() {
+            Initialize();	// allow derived classes to instantiate rules, states and nonTerminals
+
+            NextToken = 0;
+            FsaState = states[0];
+
+            StateStack.Push( FsaState );
+            valueStack.Push( CurrentSemanticValue );
+            LocationStack.Push( CurrentLocationSpan );
+
+            while (true) {
+#if TRACE_ACTIONS
+                Console.Error.WriteLine("Entering state {0} ", FsaState.number);
+                DisplayStack();
+#endif
+                int action = FsaState.defaultAction;
+
+                if (FsaState.ParserTable != null) {
+                    if (NextToken == 0) {
+                        // We save the last token span, so that the location span
+                        // of production right hand sides that begin or end with a
+                        // nullable production will be correct.
+                        LastSpan = scanner.yylloc;
+                        NextToken = scanner.yylex();
+#if TRACE_ACTIONS
+                       Console.Error.WriteLine( "Reading: Next token is {0}", TerminalToString( NextToken ) );
+#endif
+                    }
+#if TRACE_ACTIONS
+                    else 
+                        Console.Error.WriteLine( "Next token is still {0}", TerminalToString( NextToken ) );
+#endif
+                    if (FsaState.ParserTable.ContainsKey( NextToken ))
+                        action = FsaState.ParserTable[NextToken];
+                }
+
+                if (action > 0)         // shift
+                {
+                    Shift( action );
+                }
+                else if (action < 0)   // reduce
+                {
+                    try {
+                        Reduce( -action );
+                        if (action == -1)	// accept
+                            return true;
+                    }
+                    catch (AbortException) {
+                            return false;
+                    }
+                    catch (AcceptException) {
+                        return true;
+                    }
+                    catch (ErrorException) {
+                        if (!ErrorRecovery())
+                            return false;
+                        else
+                            throw;  // Rethrow x, preserving information.
+                    }
+                }
+                else if (action == 0 && !ErrorRecovery()) // error
+                    return false;
+            }
+        }
+
+        private void Shift( int stateIndex ) {
+#if TRACE_ACTIONS
+				Console.Error.Write("Shifting token {0}, ", TerminalToString(NextToken));
+#endif
+            FsaState = states[stateIndex];
+
+            valueStack.Push( scanner.yylval );
+            StateStack.Push( FsaState );
+            LocationStack.Push( scanner.yylloc );
+
+            if (recovering) {
+                if (NextToken != errorToken)
+                    tokensSinceLastError++;
+
+                if (tokensSinceLastError > 5)
+                    recovering = false;
+            }
+
+            if (NextToken != endOfFileToken)
+                NextToken = 0;
+        }
+
+        private void Reduce( int ruleNumber ) {
+#if TRACE_ACTIONS
+				DisplayRule(ruleNumber);
+#endif
+            Rule rule = rules[ruleNumber];
+            int rhLen = rule.RightHandSide.Length;
+            //
+            //  Default actions for unit productions.
+            //
+            if (rhLen == 1) {
+                CurrentSemanticValue = valueStack.TopElement();   // Default action: $$ = $1;
+                CurrentLocationSpan = LocationStack.TopElement(); // Default action "@$ = @1;
+            }
+            else if (rhLen == 0) {
+                // Create a new blank value.
+                // Explicit semantic action may mutate this value
+                CurrentSemanticValue = default( TValue );
+                // The location span for an empty production will start with the
+                // beginning of the next lexeme, and end with the finish of the
+                // previous lexeme.  This gives the correct behaviour when this
+                // nonsense value is used in later Merge operations.
+                CurrentLocationSpan = (scanner.yylloc != null && LastSpan != null ?
+                    scanner.yylloc.Merge( LastSpan ) :
+                    default( TSpan ));
+            }
+            else {
+                // Default action: $$ = $1;
+                CurrentSemanticValue = valueStack[LocationStack.Depth - rhLen];
+                //  Default action "@$ = @1.Merge(@N)" for location info.
+                TSpan at1 = LocationStack[LocationStack.Depth - rhLen];
+                TSpan atN = LocationStack[LocationStack.Depth - 1];
+                CurrentLocationSpan =
+                    ((at1 != null && atN != null) ? at1.Merge( atN ) : default( TSpan ));
+            }
+
+            DoAction( ruleNumber );
+
+            for (int i = 0; i < rule.RightHandSide.Length; i++) {
+                StateStack.Pop();
+                valueStack.Pop();
+                LocationStack.Pop();
+            }
+            FsaState = StateStack.TopElement();
+
+            if (FsaState.Goto.ContainsKey( rule.LeftHandSide ))
+                FsaState = states[FsaState.Goto[rule.LeftHandSide]];
+
+            StateStack.Push( FsaState );
+            valueStack.Push( CurrentSemanticValue );
+            LocationStack.Push( CurrentLocationSpan );
+        }
+
+        /// <summary>
+        /// Execute the selected action from array.
+        /// Must be overriden in derived classes.
+        /// </summary>
+        /// <param name="actionNumber">Index of the action to perform</param>
+        protected abstract void DoAction( int actionNumber );
+
+        private bool ErrorRecovery() {
+            bool discard;
+
+            if (!recovering) // if not recovering from previous error
+                ReportError();
+
+            if (!FindErrorRecoveryState())
+                return false;
+            //
+            //  The interim fix for the "looping in error recovery"
+            //  artifact involved moving the setting of the recovering 
+            //  bool until after invalid tokens have been discarded.
+            //
+            ShiftErrorToken();
+            discard = DiscardInvalidTokens();
+            recovering = true;
+            tokensSinceLastError = 0;
+            return discard;
+        }
+
+        private void ReportError() {
+            StringBuilder errorMsg = new StringBuilder();
+            errorMsg.AppendFormat( "Syntax error, unexpected {0}", TerminalToString( NextToken ) );
+
+            if (FsaState.ParserTable.Count < 7) {
+                bool first = true;
+                foreach (int terminal in FsaState.ParserTable.Keys) {
+                    if (first)
+                        errorMsg.Append( ", expecting " );
+                    else
+                        errorMsg.Append( ", or " );
+
+                    errorMsg.Append( TerminalToString( terminal ) );
+                    first = false;
+                }
+            }
+            scanner.yyerror( errorMsg.ToString() );
+        }
+
+        private void ShiftErrorToken() {
+            int old_next = NextToken;
+            NextToken = errorToken;
+
+            Shift( FsaState.ParserTable[NextToken] );
+
+#if TRACE_ACTIONS
+				Console.Error.WriteLine("Entering state {0} ", FsaState.number);
+#endif
+            NextToken = old_next;
+        }
+
+        private bool FindErrorRecoveryState() {
+            while (true)    // pop states until one found that accepts error token
+            {
+                if (FsaState.ParserTable != null &&
+                    FsaState.ParserTable.ContainsKey( errorToken ) &&
+                    FsaState.ParserTable[errorToken] > 0) // shift
+                    return true;
+
+#if TRACE_ACTIONS
+					Console.Error.WriteLine("Error: popping state {0}", StateStack.TopElement().number);
+#endif
+                StateStack.Pop();
+                valueStack.Pop();
+                LocationStack.Pop();
+
+#if TRACE_ACTIONS
+			    DisplayStack();
+#endif
+                if (StateStack.IsEmpty()) {
+#if TRACE_ACTIONS
+                        Console.Error.WriteLine("Aborting: didn't find a state that accepts error token");
+#endif
+                    return false;
+                }
+                else
+                    FsaState = StateStack.TopElement();
+            }
+        }
+
+        private bool DiscardInvalidTokens() {
+
+            int action = FsaState.defaultAction;
+
+            if (FsaState.ParserTable != null) {
+                // Discard tokens until find one that works ...
+                while (true) {
+                    if (NextToken == 0) {
+#if TRACE_ACTIONS
+                            Console.Error.Write("Reading a token: ");
+#endif
+                        NextToken = scanner.yylex();
+                    }
+#if TRACE_ACTIONS
+                        Console.Error.WriteLine("Next token is {0}", TerminalToString(NextToken));
+#endif
+                    if (NextToken == endOfFileToken)
+                        return false;
+
+                    if (FsaState.ParserTable.ContainsKey( NextToken ))
+                        action = FsaState.ParserTable[NextToken];
+
+                    if (action != 0)
+                        return true;
+                    else {
+#if TRACE_ACTIONS
+                            Console.Error.WriteLine("Error: Discarding {0}", TerminalToString(NextToken));
+#endif
+                        NextToken = 0;
+                    }
+                }
+            }
+            else if (recovering && tokensSinceLastError == 0) {
+                // 
+                //  Boolean recovering is not set until after the first
+                //  error token has been shifted.  Thus if we get back 
+                //  here with recovering set and no tokens read we are
+                //  looping on the same error recovery action.  This 
+                //  happens if current_state.ParserTable is null because
+                //  the state has an LR(0) reduction, but not all
+                //  lookahead tokens are valid.  This only occurs for
+                //  error productions that *end* on "error".
+                //
+                //  This action discards tokens one at a time until
+                //  the looping stops.  Another attack would be to always
+                //  use the LALR(1) table if a production ends on "error"
+                //
+#if TRACE_ACTIONS
+                    Console.Error.WriteLine("Error: panic discard of {0}", TerminalToString(NextToken));
+#endif
+                if (NextToken == endOfFileToken)
+                    return false;
+                NextToken = 0;
+                return true;
+            }
+            else
+                return true;
+        }
+
+        /// <summary>
+        /// Traditional YACC method.  Discards the next input token.
+        /// </summary>
+        [SuppressMessage( "Microsoft.Naming", "CA1709:IdentifiersShouldBeCasedCorrectly", MessageId = "yyclearin" )]
+        [SuppressMessage( "Microsoft.Naming", "CA1704:IdentifiersShouldBeSpelledCorrectly", MessageId = "yyclearin" )]
+        // Reason for FxCop message suppression -
+        // This is a traditional name for YACC-like functionality
+        protected void yyclearin() { NextToken = 0; }
+
+        /// <summary>
+        /// Tradional YACC method. Clear the "recovering" flag.
+        /// </summary>
+        [SuppressMessage( "Microsoft.Naming", "CA1709:IdentifiersShouldBeCasedCorrectly", MessageId = "yyerrok" )]
+        [SuppressMessage( "Microsoft.Naming", "CA1704:IdentifiersShouldBeSpelledCorrectly", MessageId = "yyerrok" )]
+        // Reason for FxCop message suppression -
+        // This is a traditional name for YACC-like functionality
+        protected void yyerrok() { recovering = false; }
+
+        /// <summary>
+        /// OBSOLETE FOR VERSION 1.4.0
+        /// Method used by derived types to insert new
+        /// state instances in the "states" array.
+        /// </summary>
+        /// <param name="stateNumber">index of the state</param>
+        /// <param name="state">data for the state</param>
+        protected void AddState( int stateNumber, State state ) {
+            states[stateNumber] = state;
+            state.number = stateNumber;
+        }
+
+        private void DisplayStack() {
+            Console.Error.Write( "State stack is now:" );
+            for (int i = 0; i < StateStack.Depth; i++)
+                Console.Error.Write( " {0}", StateStack[i].number );
+            Console.Error.WriteLine();
+        }
+
+        private void DisplayRule( int ruleNumber ) {
+            Console.Error.Write( "Reducing stack by rule {0}, ", ruleNumber );
+            DisplayProduction( rules[ruleNumber] );
+        }
+
+        private void DisplayProduction( Rule rule ) {
+            if (rule.RightHandSide.Length == 0)
+                Console.Error.Write( "/* empty */ " );
+            else
+                foreach (int symbol in rule.RightHandSide)
+                    Console.Error.Write( "{0} ", SymbolToString( symbol ) );
+
+            Console.Error.WriteLine( "-> {0}", SymbolToString( rule.LeftHandSide ) );
+        }
+
+        /// <summary>
+        /// Abstract state class naming terminal symbols.
+        /// This is overridden by derived classes with the
+        /// name (or alias) to be used in error messages.
+        /// </summary>
+        /// <param name="terminal">The terminal ordinal</param>
+        /// <returns></returns>
+        protected abstract string TerminalToString( int terminal );
+
+        private string SymbolToString( int symbol ) {
+            if (symbol < 0)
+                return nonTerminals[-symbol - 1];
+            else
+                return TerminalToString( symbol );
+        }
+
+        /// <summary>
+        /// Return text representation of argument character
+        /// </summary>
+        /// <param name="input">The character to convert</param>
+        /// <returns>String representation of the character</returns>
+        protected static string CharToString( char input ) {
+            switch (input) {
+                case '\a': return @"'\a'";
+                case '\b': return @"'\b'";
+                case '\f': return @"'\f'";
+                case '\n': return @"'\n'";
+                case '\r': return @"'\r'";
+                case '\t': return @"'\t'";
+                case '\v': return @"'\v'";
+                case '\0': return @"'\0'";
+                default: return string.Format( CultureInfo.InvariantCulture, "'{0}'", input );
+            }
+        }
+    }
+
+    /// <summary>
+    /// Classes implementing this interface must supply a
+    /// method that merges two location objects to return
+    /// a new object of the same type.
+    /// GPPG-generated parsers have the default location
+    /// action equivalent to "@$ = @1.Merge(@N);" where N
+    /// is the right-hand-side length of the production.
+    /// </summary>
+    /// <typeparam name="TSpan">The Location type</typeparam>
+#if EXPORT_GPPG
+    public interface IMerge<TSpan>
+#else
+    internal interface IMerge<TSpan>
+#endif
+ {
+        /// <summary>
+        /// Interface method that creates a location object from
+        /// the current and last object.  Typically used to create
+        /// a location object extending from the start of the @1
+        /// object to the end of the @N object.
+        /// </summary>
+        /// <param name="last">The lexically last object to merge</param>
+        /// <returns>The merged location object</returns>
+        TSpan Merge( TSpan last );
+    }
+
+    /// <summary>
+    /// This is the default class that carries location
+    /// information from the scanner to the parser.
+    /// If you don't declare "%YYLTYPE Foo" the parser
+    /// will expect to deal with this type.
+    /// </summary>
+#if EXPORT_GPPG
+    public class LexLocation : IMerge<LexLocation>
+#else
+    [SuppressMessage( "Microsoft.Performance", "CA1812:AvoidUninstantiatedInternalClasses" )]
+    internal class LexLocation : IMerge<LexLocation>
+#endif
+ {
+        private int startLine;   // start line
+        private int startColumn; // start column
+        private int endLine;     // end line
+        private int endColumn;   // end column
+
+        /// <summary>
+        /// The line at which the text span starts.
+        /// </summary>
+        public int StartLine { get { return startLine; } }
+
+        /// <summary>
+        /// The column at which the text span starts.
+        /// </summary>
+        public int StartColumn { get { return startColumn; } }
+
+        /// <summary>
+        /// The line on which the text span ends.
+        /// </summary>
+        public int EndLine { get { return endLine; } }
+
+        /// <summary>
+        /// The column of the first character
+        /// beyond the end of the text span.
+        /// </summary>
+        public int EndColumn { get { return endColumn; } }
+
+        /// <summary>
+        /// Default no-arg constructor.
+        /// </summary>
+        public LexLocation() { }
+
+        /// <summary>
+        /// Constructor for text-span with given start and end.
+        /// </summary>
+        /// <param name="sl">start line</param>
+        /// <param name="sc">start column</param>
+        /// <param name="el">end line </param>
+        /// <param name="ec">end column</param>
+        public LexLocation( int sl, int sc, int el, int ec ) { startLine = sl; startColumn = sc; endLine = el; endColumn = ec; }
+
+        /// <summary>
+        /// Create a text location which spans from the 
+        /// start of "this" to the end of the argument "last"
+        /// </summary>
+        /// <param name="last">The last location in the result span</param>
+        /// <returns>The merged span</returns>
+        public LexLocation Merge( LexLocation last ) { return new LexLocation( this.startLine, this.startColumn, last.endLine, last.endColumn ); }
+    }
+
+    /// <summary>
+    /// Abstract scanner class that GPPG expects its scanners to 
+    /// extend.
+    /// </summary>
+    /// <typeparam name="TValue">Semantic value type YYSTYPE</typeparam>
+    /// <typeparam name="TSpan">Source location type YYLTYPE</typeparam>
+#if EXPORT_GPPG
+    public abstract class AbstractScanner<TValue, TSpan>
+#else
+    internal abstract class AbstractScanner<TValue, TSpan>
+#endif
+ where TSpan : IMerge<TSpan> {
+        /// <summary>
+        /// Lexical value optionally set by the scanner. The value
+        /// is of the %YYSTYPE type declared in the parser spec.
+        /// </summary>
+        [SuppressMessage( "Microsoft.Design", "CA1051:DoNotDeclareVisibleInstanceFields" )]
+        [SuppressMessage( "Microsoft.Naming", "CA1709:IdentifiersShouldBeCasedCorrectly", MessageId = "yylval" )]
+        [SuppressMessage( "Microsoft.Naming", "CA1704:IdentifiersShouldBeSpelledCorrectly", MessageId = "yylval" )]
+        // Reason for FxCop message suppression -
+        // This is a traditional name for YACC-like functionality
+        // A field must be declared for this value of parametric type,
+        // since it may be instantiated by a value struct.  If it were 
+        // implemented as a property, machine generated code in derived
+        // types would not be able to select on the returned value.
+#pragma warning disable 649
+        public TValue yylval;                     // Lexical value: set by scanner
+#pragma warning restore 649
+
+        /// <summary>
+        /// Current scanner location property. The value is of the
+        /// type declared by %YYLTYPE in the parser specification.
+        /// </summary>
+        [SuppressMessage( "Microsoft.Naming", "CA1709:IdentifiersShouldBeCasedCorrectly", MessageId = "yylloc" )]
+        [SuppressMessage( "Microsoft.Naming", "CA1704:IdentifiersShouldBeSpelledCorrectly", MessageId = "yylloc" )]
+        // Reason for FxCop message suppression -
+        // This is a traditional name for YACC-like functionality
+        public virtual TSpan yylloc {
+            get { return default( TSpan ); }       // Empty implementation allowing
+            set { /* skip */ }                   // yylloc to be ignored entirely.
+        }
+
+        /// <summary>
+        /// Main call point for LEX-like scanners.  Returns an int
+        /// corresponding to the token recognized by the scanner.
+        /// </summary>
+        /// <returns>An int corresponding to the token</returns>
+        [SuppressMessage( "Microsoft.Naming", "CA1709:IdentifiersShouldBeCasedCorrectly", MessageId = "yylex" )]
+        [SuppressMessage( "Microsoft.Naming", "CA1704:IdentifiersShouldBeSpelledCorrectly", MessageId = "yylex" )]
+        // Reason for FxCop message suppression -
+        // This is a traditional name for YACC-like functionality
+        public abstract int yylex();
+
+        /// <summary>
+        /// Traditional error reporting provided by LEX-like scanners
+        /// to their YACC-like clients.
+        /// </summary>
+        /// <param name="format">Message format string</param>
+        /// <param name="args">Optional array of args</param>
+        [SuppressMessage( "Microsoft.Naming", "CA1709:IdentifiersShouldBeCasedCorrectly", MessageId = "yyerror" )]
+        [SuppressMessage( "Microsoft.Naming", "CA1704:IdentifiersShouldBeSpelledCorrectly", MessageId = "yyerror" )]
+        // Reason for FxCop message suppression -
+        // This is a traditional name for YACC-like functionality
+        public virtual void yyerror( string format, params object[] args ) { }
+    }
+
+    /// <summary>
+    /// Encapsulated state for the parser.
+    /// Opaque to users, visible to the tool-generated code.
+    /// </summary>
+#if EXPORT_GPPG
+    public class State
+    {
+        /// <summary>
+        /// The number of states in the automaton.
+        /// </summary>
+        public int number;
+#else
+    internal class State {
+        /// <summary>
+        /// The index of this state in the states array.
+        /// </summary>
+        internal int number;
+#endif
+        internal Dictionary<int, int> ParserTable;   // Terminal -> ParseAction
+        internal Dictionary<int, int> Goto;          // NonTerminal -> State;
+        internal int defaultAction; // = 0;		     // ParseAction
+
+        /// <summary>
+        /// State transition data for this state. Pairs of elements of the 
+        /// goto array associate symbol ordinals with next state indices.
+        /// The actions array is passed to another constructor. 
+        /// </summary>
+        /// <param name="actions">The action list</param>c
+        /// <param name="goToList">Next state data</param>
+        public State( int[] actions, int[] goToList )
+            : this( actions ) {
+            Goto = new Dictionary<int, int>();
+            for (int i = 0; i < goToList.Length; i += 2)
+                Goto.Add( goToList[i], goToList[i + 1] );
+        }
+
+        /// <summary>
+        /// Action data for this state. Pairs of elements of the 
+        /// action array associate action ordinals with each of
+        /// those symbols that have actions in the current state.
+        /// </summary>
+        /// <param name="actions">The action array</param>
+        public State( int[] actions ) {
+            ParserTable = new Dictionary<int, int>();
+            for (int i = 0; i < actions.Length; i += 2)
+                ParserTable.Add( actions[i], actions[i + 1] );
+        }
+
+        /// <summary>
+        /// Set the default action for this state.
+        /// </summary>
+        /// <param name="defaultAction">Ordinal of the default action</param>
+        public State( int defaultAction ) {
+            this.defaultAction = defaultAction;
+        }
+
+        /// <summary>
+        /// Set the default action and the state transition table.
+        /// </summary>
+        /// <param name="defaultAction">The default action</param>
+        /// <param name="goToList">Transitions from this state</param>
+        public State( int defaultAction, int[] goToList )
+            : this( defaultAction ) {
+            Goto = new Dictionary<int, int>();
+            for (int i = 0; i < goToList.Length; i += 2)
+                Goto.Add( goToList[i], goToList[i + 1] );
+        }
+    }
+
+    /// <summary>
+    /// Rule representation at runtime.
+    /// </summary>
+#if EXPORT_GPPG
+    public class Rule
+#else
+    internal class Rule
+#endif
+ {
+        internal int LeftHandSide; // symbol
+        internal int[] RightHandSide; // symbols
+
+        /// <summary>
+        /// Rule constructor.  This holds the ordinal of
+        /// the left hand side symbol, and the list of
+        /// right hand side symbols, in lexical order.
+        /// </summary>
+        /// <param name="left">The LHS non-terminal</param>
+        /// <param name="right">The RHS symbols, in lexical order</param>
+        public Rule( int left, int[] right ) {
+            this.LeftHandSide = left;
+            this.RightHandSide = right;
+        }
+    }
+
+    /// <summary>
+    /// Stack utility for the shift-reduce parser.
+    /// GPPG parsers have three instances:
+    /// (1) The parser state stack, T = QUT.Gppg.State,
+    /// (2) The semantic value stack, T = TValue,
+    /// (3) The location stack, T = TSpan.
+    /// </summary>
+    /// <typeparam name="T"></typeparam>
+#if EXPORT_GPPG
+    public class PushdownPrefixState<T>
+#else
+    internal class PushdownPrefixState<T>
+#endif
+ {
+        //  Note that we cannot use the BCL Stack<T> class
+        //  here as derived types need to index into stacks.
+        //
+        private T[] array = new T[8];
+        private int tos = 0;
+
+        /// <summary>
+        /// Indexer for values of the stack below the top.
+        /// </summary>
+        /// <param name="index">index of the element, starting from the bottom</param>
+        /// <returns>the selected element</returns>
+        public T this[int index] { get { return array[index]; } }
+
+        /// <summary>
+        /// The current depth of the stack.
+        /// </summary>
+        public int Depth { get { return tos; } }
+
+        internal void Push( T value ) {
+            if (tos >= array.Length) {
+                T[] newarray = new T[array.Length * 2];
+                System.Array.Copy( array, newarray, tos );
+                array = newarray;
+            }
+            array[tos++] = value;
+        }
+
+        internal T Pop() {
+            T rslt = array[--tos];
+            array[tos] = default( T );
+            return rslt;
+        }
+
+        internal T TopElement() { return array[tos - 1]; }
+
+        internal bool IsEmpty() { return tos == 0; }
+    }
+}
+

src/DataStreamSuite/Bts.ColumnExpressions/Usings.cs

@@ -0,0 +1,4 @@
+global using System;
+global using System.Linq;
+global using System.Collections.Generic;
+global using System.Collections.Immutable;

src/DataStreamSuite/Bts.ColumnSortExpressions/Bts.ColumnSortExpressions.csproj

@@ -0,0 +1,24 @@
+<Project Sdk="Microsoft.NET.Sdk">
+
+  <PropertyGroup>
+    <TargetFramework>net7.0</TargetFramework>
+    <ImplicitUsings>enable</ImplicitUsings>
+    <Nullable>enable</Nullable>
+  </PropertyGroup>
+
+  <ItemGroup>
+    <PackageReference Include="YaccLexTools" Version="1.2.2" />
+  </ItemGroup>
+
+  <ItemGroup>
+    <ProjectReference Include="..\DataStreamCore\DataStreamCore.csproj" />
+    <ProjectReference Include="..\ExpressionCore\ExpressionCore.csproj" />
+    <ProjectReference Include="..\..\client\spark_core\spark_core.csproj" />
+  </ItemGroup>
+
+  <ItemGroup Label="ColumnSortExpression">
+    <Scanner Include="ColumnSortExpressionTokens.lex" />
+    <Parser Include="ColumnSortExpressionGrammar.y" />
+  </ItemGroup>
+
+</Project>

src/DataStreamSuite/Bts.ColumnSortExpressions/ColumnSortExpression.cs

@@ -0,0 +1,11 @@
+namespace Bts.ColumnSortExpressions;
+
+public class ColumnSortExpression {
+   public ColumnSortExpression(string name, bool isAscending = true) {
+      Name = name;
+      IsAscending = isAscending;
+   }
+
+   public string Name { get; }
+   public bool IsAscending { get; }
+}

src/DataStreamSuite/Bts.ColumnSortExpressions/ColumnSortExpressionGrammar.y

@@ -0,0 +1,29 @@
+%namespace Bts.ColumnSortExpressions
+%partial
+%visibility internal
+%tokentype ColumnSortExpressionToken
+%parsertype ColumnSortExpressionParser
+
+%union {
+   public string Identifier;
+   public ColumnSortExpression Column;
+   public ImmutableList<ColumnSortExpression> Columns;
+}
+
+%token ASCENDING, DESCENDING, IDENTIFIER
+
+%type <Column> Column
+%type <Columns> Columns
+
+%%
+
+Columns :
+   Column { $$ = ImmutableList.CreateRange(new[] { $1 }); }
+   | Columns Column { $$ = $1.Add($2); }
+   ;
+
+Column :
+   IDENTIFIER { $$ = new($1.Identifier, true); }
+   | ASCENDING IDENTIFIER { $$ = new($2.Identifier, true); }
+   | DESCENDING IDENTIFIER { $$ = new($2.Identifier, false); }
+   ;

src/DataStreamSuite/Bts.ColumnSortExpressions/ColumnSortExpressionInterpreter.cs

@@ -0,0 +1,18 @@
+namespace Bts.ColumnSortExpressions;
+
+using System.IO;
+using System.Text;
+
+public class ColumnSortExpressionInterpreter {
+   public static IList<ColumnSortExpression> Evaluate(Stream stream) {
+      var parser = new ColumnSortExpressionParser();
+      var res = parser.Parse(stream);
+      return res?.Columns ?? ImmutableList<ColumnSortExpression>.Empty.Add(
+         res?.Column ?? throw new Exception("Failed to parse column(s) expression"));
+   }
+
+   public static IList<ColumnSortExpression> Evaluate(string str) {
+      using var ms = new MemoryStream(Encoding.Default.GetBytes(str));
+      return Evaluate(ms);
+   }
+}

src/DataStreamSuite/Bts.ColumnSortExpressions/ColumnSortExpressionParser.cs

@@ -0,0 +1,18 @@
+namespace Bts.ColumnSortExpressions;
+
+using System.Text;
+
+partial class ColumnSortExpressionParser {
+   public ColumnSortExpressionParser() : base(null) { }
+
+   public ValueType? Parse(string s) {
+      using var ms = new MemoryStream(Encoding.Default.GetBytes(s));
+      return Parse(ms);
+   }
+
+   public ValueType? Parse(Stream stream) {
+      Scanner = new ColumnSortExpressionScanner(stream);
+      Parse();
+      return CurrentSemanticValue;
+   }
+}

src/DataStreamSuite/Bts.ColumnSortExpressions/ColumnSortExpressionScanner.cs

@@ -0,0 +1,9 @@
+namespace Bts.ColumnSortExpressions;
+
+partial class ColumnSortExpressionScanner {
+   public override void yyerror(string format, params object[] args) {
+      base.yyerror(format, args);
+      Console.WriteLine(format, args);
+      Console.WriteLine();
+   }
+}

src/DataStreamSuite/Bts.ColumnSortExpressions/ColumnSortExpressionTokens.lex

@@ -0,0 +1,30 @@
+%namespace Bts.ColumnSortExpressions
+%scannertype ColumnSortExpressionScanner
+%visibility internal
+%tokentype ColumnSortExpressionToken
+
+%option stack, minimize, parser, verbose, persistbuffer, noembedbuffers 
+
+/* Literal Constants */
+Letter                   [a-zA-Z]
+
+Identifier               {Letter}({Letter}*[0-9]*_*)*
+Ascending                +
+Descending               -
+
+%%
+
+{Ascending} {
+   return (int)ColumnSortExpressionToken.ASCENDING;
+}
+
+{Descending} {
+   return (int)ColumnSortExpressionToken.DESCENDING;
+}
+
+{Identifier} {
+   yylval.Identifier = yytext;
+   return (int)ColumnSortExpressionToken.IDENTIFIER;
+}
+
+%%

src/DataStreamSuite/Bts.ColumnSortExpressions/GplexBuffers.cs

@@ -0,0 +1,720 @@
+// ==============================================================
+// <auto-generated>
+// This code automatically produced from an embedded resource.
+// Do not edit this file, or it will become incompatible with 
+// the specification from which it was generated.
+// </auto-generated>
+// ==============================================================
+
+using System;
+using System.IO;
+using System.Text;
+using System.Collections.Generic;
+using System.Diagnostics.CodeAnalysis;
+using System.Runtime.Serialization;
+using System.Globalization;
+
+namespace QUT.GplexBuffers
+{
+// Code copied from GPLEX embedded resource
+    [Serializable]
+    internal class BufferException : Exception
+    {
+        public BufferException() { }
+        public BufferException(string message) : base(message) { }
+        public BufferException(string message, Exception innerException)
+            : base(message, innerException) { }
+        protected BufferException(SerializationInfo info, StreamingContext context)
+            : base(info, context) { }
+    }
+
+    internal abstract class ScanBuff
+    {
+        private string fileNm;
+
+        public const int EndOfFile = -1;
+        public const int UnicodeReplacementChar = 0xFFFD;
+
+        public bool IsFile { get { return (fileNm != null); } }
+        public string FileName { get { return fileNm; } set { fileNm = value; } }
+
+        public abstract int Pos { get; set; }
+        public abstract int Read();
+        public virtual void Mark() { }
+
+        public abstract string GetString(int begin, int limit);
+
+        public static ScanBuff GetBuffer(string source)
+        {
+            return new StringBuffer(source);
+        }
+
+        public static ScanBuff GetBuffer(IList<string> source)
+        {
+            return new LineBuffer(source);
+        }
+
+#if (!NOFILES)
+        public static ScanBuff GetBuffer(Stream source)
+        {
+            return new BuildBuffer(source);
+        }
+
+#if (!BYTEMODE)
+        public static ScanBuff GetBuffer(Stream source, int fallbackCodePage)
+        {
+            return new BuildBuffer(source, fallbackCodePage);
+        }
+#endif // !BYTEMODE
+#endif // !NOFILES
+    }
+
+    #region Buffer classes
+
+    // ==============================================================
+    // =====  Definitions for various ScanBuff derived classes   ====
+    // ==============================================================
+    // ===============         String input          ================
+    // ==============================================================
+
+    /// <summary>
+    /// This class reads characters from a single string as
+    /// required, for example, by Visual Studio language services
+    /// </summary>
+    sealed class StringBuffer : ScanBuff
+    {
+        string str;        // input buffer
+        int bPos;          // current position in buffer
+        int sLen;
+
+        public StringBuffer(string source)
+        {
+            this.str = source;
+            this.sLen = source.Length;
+            this.FileName = null;
+        }
+
+        public override int Read()
+        {
+            if (bPos < sLen) return str[bPos++];
+            else if (bPos == sLen) { bPos++; return '\n'; }   // one strike, see new line
+            else { bPos++; return EndOfFile; }                // two strikes and you're out!
+        }
+
+        public override string GetString(int begin, int limit)
+        {
+            //  "limit" can be greater than sLen with the BABEL
+            //  option set.  Read returns a "virtual" EOL if
+            //  an attempt is made to read past the end of the
+            //  string buffer.  Without the guard any attempt 
+            //  to fetch yytext for a token that includes the 
+            //  EOL will throw an index exception.
+            if (limit > sLen) limit = sLen;
+            if (limit <= begin) return "";
+            else return str.Substring(begin, limit - begin);
+        }
+
+        public override int Pos
+        {
+            get { return bPos; }
+            set { bPos = value; }
+        }
+
+        public override string ToString() { return "StringBuffer"; }
+    }
+
+    // ==============================================================
+    //  The LineBuff class contributed by Nigel Horspool, 
+    //  nigelh@cs.uvic.cs
+    // ==============================================================
+
+    sealed class LineBuffer : ScanBuff
+    {
+        IList<string> line;    // list of source lines from a file
+        int numLines;          // number of strings in line list
+        string curLine;        // current line in that list
+        int cLine;             // index of current line in the list
+        int curLen;            // length of current line
+        int curLineStart;      // position of line start in whole file
+        int curLineEnd;        // position of line end in whole file
+        int maxPos;            // max position ever visited in whole file
+        int cPos;              // ordinal number of code in source
+
+        // Constructed from a list of strings, one per source line.
+        // The lines have had trailing '\n' characters removed.
+        public LineBuffer(IList<string> lineList)
+        {
+            line = lineList;
+            numLines = line.Count;
+            cPos = curLineStart = 0;
+            curLine = (numLines > 0 ? line[0] : "");
+            maxPos = curLineEnd = curLen = curLine.Length;
+            cLine = 1;
+            FileName = null;
+        }
+
+        public override int Read()
+        {
+            if (cPos < curLineEnd)
+                return curLine[cPos++ - curLineStart];
+            if (cPos++ == curLineEnd)
+                return '\n';
+            if (cLine >= numLines)
+                return EndOfFile;
+            curLine = line[cLine];
+            curLen = curLine.Length;
+            curLineStart = curLineEnd + 1;
+            curLineEnd = curLineStart + curLen;
+            if (curLineEnd > maxPos)
+                maxPos = curLineEnd;
+            cLine++;
+            return curLen > 0 ? curLine[0] : '\n';
+        }
+
+        // To speed up searches for the line containing a position
+        private int cachedPosition;
+        private int cachedIxdex;
+        private int cachedLineStart;
+
+        // Given a position pos within the entire source, the results are
+        //   ix     -- the index of the containing line
+        //   lstart -- the position of the first character on that line
+        private void findIndex(int pos, out int ix, out int lstart)
+        {
+            if (pos >= cachedPosition)
+            {
+                ix = cachedIxdex; lstart = cachedLineStart;
+            }
+            else
+            {
+                ix = lstart = 0;
+            }
+            while (ix < numLines)
+            {
+                int len = line[ix].Length + 1;
+                if (pos < lstart + len) break;
+                lstart += len;
+                ix++;
+            }
+            cachedPosition = pos;
+            cachedIxdex = ix;
+            cachedLineStart = lstart;
+        }
+
+        public override string GetString(int begin, int limit)
+        {
+            if (begin >= maxPos || limit <= begin) return "";
+            int endIx, begIx, endLineStart, begLineStart;
+            findIndex(begin, out begIx, out begLineStart);
+            int begCol = begin - begLineStart;
+            findIndex(limit, out endIx, out endLineStart);
+            int endCol = limit - endLineStart;
+            string s = line[begIx];
+            if (begIx == endIx)
+            {
+                // the usual case, substring all on one line
+                return (endCol <= s.Length) ?
+                    s.Substring(begCol, endCol - begCol)
+                    : s.Substring(begCol) + "\n";
+            }
+            // the string spans multiple lines, yuk!
+            StringBuilder sb = new StringBuilder();
+            if (begCol < s.Length)
+                sb.Append(s.Substring(begCol));
+            for (; ; )
+            {
+                sb.Append("\n");
+                s = line[++begIx];
+                if (begIx >= endIx) break;
+                sb.Append(s);
+            }
+            if (endCol <= s.Length)
+            {
+                sb.Append(s.Substring(0, endCol));
+            }
+            else
+            {
+                sb.Append(s);
+                sb.Append("\n");
+            }
+            return sb.ToString();
+        }
+
+        public override int Pos
+        {
+            get { return cPos; }
+            set
+            {
+                cPos = value;
+                findIndex(cPos, out cLine, out curLineStart);
+                // cLine should be the *next* line after curLine.
+                curLine = (cLine < numLines ? line[cLine++] : "");
+                curLineEnd = curLineStart + curLine.Length;
+            }
+        }
+
+        public override string ToString() { return "LineBuffer"; }
+    }
+
+#if (!NOFILES)
+    // ==============================================================
+    // =====     class BuildBuff : for unicode text files    ========
+    // ==============================================================
+
+    class BuildBuffer : ScanBuff
+    {
+        // Double buffer for char stream.
+        class BufferElement
+        {
+            StringBuilder bldr = new StringBuilder();
+            StringBuilder next = new StringBuilder();
+            int minIx;
+            int maxIx;
+            int brkIx;
+            bool appendToNext;
+
+            internal BufferElement() { }
+
+            internal int MaxIndex { get { return maxIx; } }
+            // internal int MinIndex { get { return minIx; } }
+
+            internal char this[int index]
+            {
+                get
+                {
+                    if (index < minIx || index >= maxIx)
+                        throw new BufferException("Index was outside data buffer");
+                    else if (index < brkIx)
+                        return bldr[index - minIx];
+                    else
+                        return next[index - brkIx];
+                }
+            }
+
+            internal void Append(char[] block, int count)
+            {
+                maxIx += count;
+                if (appendToNext)
+                    this.next.Append(block, 0, count);
+                else
+                {
+                    this.bldr.Append(block, 0, count);
+                    brkIx = maxIx;
+                    appendToNext = true;
+                }
+            }
+
+            internal string GetString(int start, int limit)
+            {
+                if (limit <= start)
+                    return "";
+                if (start >= minIx && limit <= maxIx)
+                    if (limit < brkIx) // String entirely in bldr builder
+                        return bldr.ToString(start - minIx, limit - start);
+                    else if (start >= brkIx) // String entirely in next builder
+                        return next.ToString(start - brkIx, limit - start);
+                    else // Must do a string-concatenation
+                        return
+                            bldr.ToString(start - minIx, brkIx - start) +
+                            next.ToString(0, limit - brkIx);
+                else
+                    throw new BufferException("String was outside data buffer");
+            }
+
+            internal void Mark(int limit)
+            {
+                if (limit > brkIx + 16) // Rotate blocks
+                {
+                    StringBuilder temp = bldr;
+                    bldr = next;
+                    next = temp;
+                    next.Length = 0;
+                    minIx = brkIx;
+                    brkIx = maxIx;
+                }
+            }
+        }
+
+        BufferElement data = new BufferElement();
+
+        int bPos;            // Postion index in the StringBuilder
+        BlockReader NextBlk; // Delegate that serves char-arrays;
+
+        private string EncodingName
+        {
+            get
+            {
+                StreamReader rdr = NextBlk.Target as StreamReader;
+                return (rdr == null ? "raw-bytes" : rdr.CurrentEncoding.BodyName);
+            }
+        }
+
+        public BuildBuffer(Stream stream)
+        {
+            FileStream fStrm = (stream as FileStream);
+            if (fStrm != null) FileName = fStrm.Name;
+            NextBlk = BlockReaderFactory.Raw(stream);
+        }
+
+#if (!BYTEMODE)
+        public BuildBuffer(Stream stream, int fallbackCodePage)
+        {
+            FileStream fStrm = (stream as FileStream);
+            if (fStrm != null) FileName = fStrm.Name;
+            NextBlk = BlockReaderFactory.Get(stream, fallbackCodePage);
+        }
+#endif
+
+        /// <summary>
+        /// Marks a conservative lower bound for the buffer,
+        /// allowing space to be reclaimed.  If an application 
+        /// needs to call GetString at arbitrary past locations 
+        /// in the input stream, Mark() is not called.
+        /// </summary>
+        public override void Mark() { data.Mark(bPos - 2); }
+
+        public override int Pos
+        {
+            get { return bPos; }
+            set { bPos = value; }
+        }
+
+
+        /// <summary>
+        /// Read returns the ordinal number of the next char, or 
+        /// EOF (-1) for an end of stream.  Note that the next
+        /// code point may require *two* calls of Read().
+        /// </summary>
+        /// <returns></returns>
+        public override int Read()
+        {
+            //
+            //  Characters at positions 
+            //  [data.offset, data.offset + data.bldr.Length)
+            //  are available in data.bldr.
+            //
+            if (bPos < data.MaxIndex)
+            {
+                // ch0 cannot be EOF
+                return (int)data[bPos++];
+            }
+            else // Read from underlying stream
+            {
+                // Experimental code, blocks of page size
+                char[] chrs = new char[4096];
+                int count = NextBlk(chrs, 0, 4096);
+                if (count == 0)
+                    return EndOfFile;
+                else
+                {
+                    data.Append(chrs, count);
+                    return (int)data[bPos++];
+                }
+            }
+        }
+
+        public override string GetString(int begin, int limit)
+        {
+            return data.GetString(begin, limit);
+        }
+
+        public override string ToString()
+        {
+            return "StringBuilder buffer, encoding: " + this.EncodingName;
+        }
+    }
+
+    // =============== End ScanBuff-derived classes ==================
+
+    internal delegate int BlockReader(char[] block, int index, int number);
+
+    // A delegate factory, serving up a delegate that
+    // reads a block of characters from the underlying
+    // encoded stream, via a StreamReader object.
+    //
+    internal static class BlockReaderFactory
+    {
+        public static BlockReader Raw(Stream stream)
+        {
+            return delegate(char[] block, int index, int number)
+            {
+                byte[] b = new byte[number];
+                int count = stream.Read(b, 0, number);
+                int i = 0;
+                int j = index;
+                for (; i < count; i++, j++)
+                    block[j] = (char)b[i];
+                return count;
+            };
+        }
+
+#if (!BYTEMODE)
+        public static BlockReader Get(Stream stream, int fallbackCodePage)
+        {
+            Encoding encoding;
+            int preamble = Preamble(stream);
+
+            if (preamble != 0)  // There is a valid BOM here!
+                encoding = Encoding.GetEncoding(preamble);
+            else if (fallbackCodePage == -1) // Fallback is "raw" bytes
+                return Raw(stream);
+            else if (fallbackCodePage != -2) // Anything but "guess"
+                encoding = Encoding.GetEncoding(fallbackCodePage);
+            else // This is the "guess" option
+            {
+                int guess = new Guesser(stream).GuessCodePage();
+                stream.Seek(0, SeekOrigin.Begin);
+                if (guess == -1) // ==> this is a 7-bit file
+                    encoding = Encoding.ASCII;
+                else if (guess == 65001)
+                    encoding = Encoding.UTF8;
+                else             // ==> use the machine default
+                    encoding = Encoding.Default;
+            }
+            StreamReader reader = new StreamReader(stream, encoding);
+            return reader.Read;
+        }
+
+        static int Preamble(Stream stream)
+        {
+            int b0 = stream.ReadByte();
+            int b1 = stream.ReadByte();
+
+            if (b0 == 0xfe && b1 == 0xff)
+                return 1201; // UTF16BE
+            if (b0 == 0xff && b1 == 0xfe)
+                return 1200; // UTF16LE
+
+            int b2 = stream.ReadByte();
+            if (b0 == 0xef && b1 == 0xbb && b2 == 0xbf)
+                return 65001; // UTF8
+            //
+            // There is no unicode preamble, so we
+            // return denoter for the machine default.
+            //
+            stream.Seek(0, SeekOrigin.Begin);
+            return 0;
+        }
+#endif // !BYTEMODE
+    }
+#endif // !NOFILES
+    #endregion Buffer classes
+
+    // ==============================================================
+    // ============      class CodePageHandling         =============
+    // ==============================================================
+#if (!NOFILES)
+    public static class CodePageHandling
+    {
+        public static int GetCodePage(string option)
+        {
+            string command = option.ToUpperInvariant();
+            if (command.StartsWith("CodePage:", StringComparison.OrdinalIgnoreCase))
+                command = command.Substring(9);
+            try
+            {
+                if (command.Equals("RAW"))
+                    return -1;
+                else if (command.Equals("GUESS"))
+                    return -2;
+                else if (command.Equals("DEFAULT"))
+                    return 0;
+                else if (char.IsDigit(command[0]))
+                    return int.Parse(command, CultureInfo.InvariantCulture);
+                else
+                {
+                    Encoding enc = Encoding.GetEncoding(command);
+                    return enc.CodePage;
+                }
+            }
+            catch (FormatException)
+            {
+                Console.Error.WriteLine(
+                    "Invalid format \"{0}\", using machine default", option);
+            }
+            catch (ArgumentException)
+            {
+                Console.Error.WriteLine(
+                    "Unknown code page \"{0}\", using machine default", option);
+            }
+            return 0;
+        }
+    }
+#region guesser
+#if (!BYTEMODE)
+    // ==============================================================
+    // ============          Encoding Guesser           =============
+    // ==============================================================
+
+    /// <summary>
+    /// This class provides a simple finite state automaton that
+    /// scans the file looking for (1) valid UTF-8 byte patterns,
+    /// (2) bytes >= 0x80 which are not part of a UTF-8 sequence.
+    /// The method then guesses whether it is UTF-8 or maybe some 
+    /// local machine default encoding.  This works well for the
+    /// various Latin encodings.
+    /// </summary>
+    internal class Guesser
+    {
+        ScanBuff buffer;
+
+        public int GuessCodePage() { return Scan(); }
+
+        const int maxAccept = 10;
+        const int initial = 0;
+        const int eofNum = 0;
+        const int goStart = -1;
+        const int INITIAL = 0;
+        const int EndToken = 0;
+
+        #region user code
+        /* 
+         *  Reads the bytes of a file to determine if it is 
+         *  UTF-8 or a single-byte code page file.
+         */
+        public long utfX;
+        public long uppr;
+        #endregion user code
+
+        int state;
+        int currentStart = startState[0];
+        int code;
+
+        #region ScannerTables
+        static int[] startState = new int[] { 11, 0 };
+
+        #region CharacterMap
+        static sbyte[] map = new sbyte[256] {
+/*     '\0' */ 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 
+/*   '\x10' */ 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 
+/*   '\x20' */ 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 
+/*      '0' */ 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 
+/*      '@' */ 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 
+/*      'P' */ 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 
+/*      '`' */ 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 
+/*      'p' */ 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 
+/*   '\x80' */ 2, 2, 2, 2, 2, 2, 2, 2, 2, 2, 2, 2, 2, 2, 2, 2, 
+/*   '\x90' */ 2, 2, 2, 2, 2, 2, 2, 2, 2, 2, 2, 2, 2, 2, 2, 2, 
+/*   '\xA0' */ 2, 2, 2, 2, 2, 2, 2, 2, 2, 2, 2, 2, 2, 2, 2, 2, 
+/*   '\xB0' */ 2, 2, 2, 2, 2, 2, 2, 2, 2, 2, 2, 2, 2, 2, 2, 2, 
+/*   '\xC0' */ 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 
+/*   '\xD0' */ 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 
+/*   '\xE0' */ 3, 3, 3, 3, 3, 3, 3, 3, 3, 3, 3, 3, 3, 3, 3, 3, 
+/*   '\xF0' */ 4, 4, 4, 4, 4, 4, 4, 4, 5, 5, 5, 5, 5, 5, 5, 5 };
+        #endregion
+
+        static sbyte[][] nextState = new sbyte[][] {
+            new sbyte[] {0, 0, 0, 0, 0, 0},
+            new sbyte[] {-1, -1, 10, -1, -1, -1},
+            new sbyte[] {-1, -1, -1, -1, -1, -1},
+            new sbyte[] {-1, -1, 8, -1, -1, -1},
+            new sbyte[] {-1, -1, 5, -1, -1, -1},
+            new sbyte[] {-1, -1, 6, -1, -1, -1},
+            new sbyte[] {-1, -1, 7, -1, -1, -1},
+            null,
+            new sbyte[] {-1, -1, 9, -1, -1, -1},
+            null,
+            null,
+            new sbyte[] {-1, 1, 2, 3, 4, 2}
+        };
+
+
+        [SuppressMessage("Microsoft.Performance", "CA1810:InitializeReferenceTypeStaticFieldsInline")]
+        // Reason for suppression: cannot have self-reference in array initializer.
+        static Guesser()
+        {
+            nextState[7] = nextState[2];
+            nextState[9] = nextState[2];
+            nextState[10] = nextState[2];
+        }
+
+        int NextState()
+        {
+            if (code == ScanBuff.EndOfFile)
+                return eofNum;
+            else
+                return nextState[state][map[code]];
+        }
+        #endregion
+
+        public Guesser(System.IO.Stream file) { SetSource(file); }
+
+        public void SetSource(System.IO.Stream source)
+        {
+            this.buffer = new BuildBuffer(source);
+            code = buffer.Read();
+        }
+
+        int Scan()
+        {
+            for (; ; )
+            {
+                int next;
+                state = currentStart;
+                while ((next = NextState()) == goStart)
+                    code = buffer.Read();
+
+                state = next;
+                code = buffer.Read();
+
+                while ((next = NextState()) > eofNum)
+                {
+                    state = next;
+                    code = buffer.Read();
+                }
+                if (state <= maxAccept)
+                {
+                    #region ActionSwitch
+#pragma warning disable 162
+                    switch (state)
+                    {
+                        case eofNum:
+                            switch (currentStart)
+                            {
+                                case 11:
+                                    if (utfX == 0 && uppr == 0) return -1; /* raw ascii */
+                                    else if (uppr * 10 > utfX) return 0;   /* default code page */
+                                    else return 65001;                     /* UTF-8 encoding */
+                                    break;
+                            }
+                            return EndToken;
+                        case 1: // Recognized '{Upper128}',	Shortest string "\xC0"
+                        case 2: // Recognized '{Upper128}',	Shortest string "\x80"
+                        case 3: // Recognized '{Upper128}',	Shortest string "\xE0"
+                        case 4: // Recognized '{Upper128}',	Shortest string "\xF0"
+                            uppr++;
+                            break;
+                        case 5: // Recognized '{Utf8pfx4}{Utf8cont}',	Shortest string "\xF0\x80"
+                            uppr += 2;
+                            break;
+                        case 6: // Recognized '{Utf8pfx4}{Utf8cont}{2}',	Shortest string "\xF0\x80\x80"
+                            uppr += 3;
+                            break;
+                        case 7: // Recognized '{Utf8pfx4}{Utf8cont}{3}',	Shortest string "\xF0\x80\x80\x80"
+                            utfX += 3;
+                            break;
+                        case 8: // Recognized '{Utf8pfx3}{Utf8cont}',	Shortest string "\xE0\x80"
+                            uppr += 2;
+                            break;
+                        case 9: // Recognized '{Utf8pfx3}{Utf8cont}{2}',	Shortest string "\xE0\x80\x80"
+                            utfX += 2;
+                            break;
+                        case 10: // Recognized '{Utf8pfx2}{Utf8cont}',	Shortest string "\xC0\x80"
+                            utfX++;
+                            break;
+                        default:
+                            break;
+                    }
+#pragma warning restore 162
+                    #endregion
+                }
+            }
+        }
+    } // end class Guesser
+    
+#endif // !BYTEMODE
+#endregion
+#endif // !NOFILES
+
+// End of code copied from embedded resource
+}

src/DataStreamSuite/Bts.ColumnSortExpressions/ShiftReduceParserCode.cs

@@ -0,0 +1,884 @@
+// ==============================================================
+// <auto-generated>
+// This code automatically produced from an embedded resource.
+// Do not edit this file, or it will become incompatible with 
+// the specification from which it was generated.
+// </auto-generated>
+// ==============================================================
+
+// Gardens Point Parser Generator
+// Copyright (c) Wayne Kelly, QUT 2005-2010
+// (see accompanying GPPGcopyright.rtf)
+
+using System;
+using System.Text;
+using System.Globalization;
+using System.Collections.Generic;
+using System.Runtime.Serialization;
+using System.Diagnostics.CodeAnalysis;
+
+namespace QUT.Gppg {
+    /// <summary>
+    /// Abstract class for GPPG shift-reduce parsers.
+    /// Parsers generated by GPPG derive from this base
+    /// class, overriding the abstract Initialize() and
+    /// DoAction() methods.
+    /// </summary>
+    /// <typeparam name="TValue">Semantic value type</typeparam>
+    /// <typeparam name="TSpan">Location type</typeparam>
+#if EXPORT_GPPG
+    public abstract class ShiftReduceParser<TValue, TSpan>
+#else
+    internal abstract class ShiftReduceParser<TValue, TSpan>
+#endif
+ where TSpan : IMerge<TSpan>, new() {
+        private AbstractScanner<TValue, TSpan> scanner;
+        /// <summary>
+        /// The abstract scanner for this parser.
+        /// </summary>
+        protected AbstractScanner<TValue, TSpan> Scanner {
+            get { return scanner; }
+            set { scanner = value; }
+        }
+
+        /// <summary>
+        /// Constructor for base class
+        /// </summary>
+        /// <param name="scanner">Scanner instance for this parser</param>
+        protected ShiftReduceParser( AbstractScanner<TValue, TSpan> scanner ) {
+            this.scanner = scanner;
+        }
+
+        // ==============================================================
+        //                    TECHNICAL EXPLANATION.
+        //   Why the next two fields are not exposed via properties.
+        // ==============================================================
+        // These fields are of the generic parameter types, and are
+        // frequently instantiated as struct types in derived classes.
+        // Semantic actions are defined in the derived classes and refer
+        // to instance fields of these structs.  In such cases the code
+        // "get_CurrentSemanticValue().myField = blah;" will fail since
+        // the getter pushes the value of the field, not the reference.
+        // So, in the presence of properties, gppg would need to encode
+        // such field accesses as ... 
+        //  "tmp = get_CurrentSemanticValue(); // Fetch value
+        //   tmp.myField = blah;               // update
+        //   set_CurrentSemanticValue(tmp); "  // Write update back.
+        // There is no issue if TValue is restricted to be a ref type.
+        // The same explanation applies to scanner.yylval.
+        // ==============================================================
+        /// <summary>
+        /// The current value of the "$$" symbolic variable in the parser
+        /// </summary>
+        [SuppressMessage( "Microsoft.Design", "CA1051:DoNotDeclareVisibleInstanceFields" )]
+        protected TValue CurrentSemanticValue;
+
+        /// <summary>
+        /// The current value of the "@$" symbolic variable in the parser
+        /// </summary>
+        [SuppressMessage( "Microsoft.Design", "CA1051:DoNotDeclareVisibleInstanceFields" )]
+        protected TSpan CurrentLocationSpan;
+        protected int NextToken;
+
+        private TSpan LastSpan;
+        private State FsaState;
+        private bool recovering;
+        private int tokensSinceLastError;
+
+        private PushdownPrefixState<State> StateStack = new PushdownPrefixState<State>();
+        private PushdownPrefixState<TValue> valueStack = new PushdownPrefixState<TValue>();
+        private PushdownPrefixState<TSpan> locationStack = new PushdownPrefixState<TSpan>();
+
+        /// <summary>
+        /// The stack of semantic value (YYSTYPE) values.
+        /// </summary>
+        protected PushdownPrefixState<TValue> ValueStack { get { return valueStack; } }
+
+        /// <summary>
+        /// The stack of location value (YYLTYPE) varlues.
+        /// </summary>
+        protected PushdownPrefixState<TSpan> LocationStack { get { return locationStack; } }
+
+        private int errorToken;
+        private int endOfFileToken;
+        private string[] nonTerminals;
+        private State[] states;
+        private Rule[] rules;
+
+        /// <summary>
+        /// Initialization method to allow derived classes
+        /// to insert the rule list into this base class.
+        /// </summary>
+        /// <param name="rules">The array of Rule objects</param>
+        protected void InitRules( Rule[] rules ) { this.rules = rules; }
+
+        /// <summary>
+        /// Initialization method to allow derived classes to
+        /// insert the states table into this base class.
+        /// </summary>
+        /// <param name="states">The pre-initialized states table</param>
+        protected void InitStates( State[] states ) { this.states = states; }
+
+        /// <summary>
+        /// OBSOLETE FOR VERSION 1.4.0
+        /// </summary>
+        /// <param name="size"></param>
+        protected void InitStateTable( int size ) { states = new State[size]; }
+
+        /// <summary>
+        /// Initialization method to allow derived classes
+        /// to insert the special value for the error and EOF tokens.
+        /// </summary>
+        /// <param name="err">The error state ordinal</param>
+        /// <param name="end">The EOF stat ordinal</param>
+        protected void InitSpecialTokens( int err, int end ) {
+            errorToken = err;
+            endOfFileToken = end;
+        }
+
+        /// <summary>
+        /// Initialization method to allow derived classes to
+        /// insert the non-terminal symbol names into this base class.
+        /// </summary>
+        /// <param name="names">Non-terminal symbol names</param>
+        protected void InitNonTerminals( string[] names ) { nonTerminals = names; }
+
+        #region YYAbort, YYAccept etcetera.
+        [Serializable]
+        [SuppressMessage( "Microsoft.Design", "CA1064:ExceptionsShouldBePublic" )]
+        // Reason for FxCop message suppression -
+        // This exception cannot escape from the local context
+        private class AcceptException : Exception {
+            internal AcceptException() { }
+            protected AcceptException( SerializationInfo i, StreamingContext c ) : base( i, c ) { }
+        }
+        [Serializable]
+        [SuppressMessage( "Microsoft.Design", "CA1064:ExceptionsShouldBePublic" )]
+        // Reason for FxCop message suppression -
+        // This exception cannot escape from the local context
+        private class AbortException : Exception {
+            internal AbortException() { }
+            protected AbortException( SerializationInfo i, StreamingContext c ) : base( i, c ) { }
+        }
+        [Serializable]
+        [SuppressMessage( "Microsoft.Design", "CA1064:ExceptionsShouldBePublic" )]
+        // Reason for FxCop message suppression -
+        // This exception cannot escape from the local context
+        private class ErrorException : Exception {
+            internal ErrorException() { }
+            protected ErrorException( SerializationInfo i, StreamingContext c ) : base( i, c ) { }
+        }
+
+        // The following methods are only called from within
+        // a semantic action. The thrown exceptions can never
+        // propagate outside the ShiftReduceParser class in 
+        // which they are nested.
+
+        /// <summary>
+        /// Force parser to terminate, returning "true"
+        /// </summary>
+        protected static void YYAccept() { throw new AcceptException(); }
+
+        /// <summary>
+        /// Force parser to terminate, returning "false"
+        /// </summary>
+        protected static void YYAbort() { throw new AbortException(); }
+
+        /// <summary>
+        /// Force parser to terminate, returning
+        /// "false" if error recovery fails.
+        /// </summary>
+        protected static void YYError() { throw new ErrorException(); }
+
+        /// <summary>
+        /// Check if parser in error recovery state.
+        /// </summary>
+        protected bool YYRecovering { get { return recovering; } }
+        #endregion
+
+        /// <summary>
+        /// Abstract base method. ShiftReduceParser calls this
+        /// to initialize the base class data structures.  Concrete
+        /// parser classes must override this method.
+        /// </summary>
+        protected abstract void Initialize();
+
+        /// <summary>
+        /// Main entry point of the Shift-Reduce Parser.
+        /// </summary>
+        /// <returns>True if parse succeeds, else false for
+        /// unrecoverable errors</returns>
+        public bool Parse() {
+            Initialize();	// allow derived classes to instantiate rules, states and nonTerminals
+
+            NextToken = 0;
+            FsaState = states[0];
+
+            StateStack.Push( FsaState );
+            valueStack.Push( CurrentSemanticValue );
+            LocationStack.Push( CurrentLocationSpan );
+
+            while (true) {
+#if TRACE_ACTIONS
+                Console.Error.WriteLine("Entering state {0} ", FsaState.number);
+                DisplayStack();
+#endif
+                int action = FsaState.defaultAction;
+
+                if (FsaState.ParserTable != null) {
+                    if (NextToken == 0) {
+                        // We save the last token span, so that the location span
+                        // of production right hand sides that begin or end with a
+                        // nullable production will be correct.
+                        LastSpan = scanner.yylloc;
+                        NextToken = scanner.yylex();
+#if TRACE_ACTIONS
+                       Console.Error.WriteLine( "Reading: Next token is {0}", TerminalToString( NextToken ) );
+#endif
+                    }
+#if TRACE_ACTIONS
+                    else 
+                        Console.Error.WriteLine( "Next token is still {0}", TerminalToString( NextToken ) );
+#endif
+                    if (FsaState.ParserTable.ContainsKey( NextToken ))
+                        action = FsaState.ParserTable[NextToken];
+                }
+
+                if (action > 0)         // shift
+                {
+                    Shift( action );
+                }
+                else if (action < 0)   // reduce
+                {
+                    try {
+                        Reduce( -action );
+                        if (action == -1)	// accept
+                            return true;
+                    }
+                    catch (AbortException) {
+                            return false;
+                    }
+                    catch (AcceptException) {
+                        return true;
+                    }
+                    catch (ErrorException) {
+                        if (!ErrorRecovery())
+                            return false;
+                        else
+                            throw;  // Rethrow x, preserving information.
+                    }
+                }
+                else if (action == 0 && !ErrorRecovery()) // error
+                    return false;
+            }
+        }
+
+        private void Shift( int stateIndex ) {
+#if TRACE_ACTIONS
+				Console.Error.Write("Shifting token {0}, ", TerminalToString(NextToken));
+#endif
+            FsaState = states[stateIndex];
+
+            valueStack.Push( scanner.yylval );
+            StateStack.Push( FsaState );
+            LocationStack.Push( scanner.yylloc );
+
+            if (recovering) {
+                if (NextToken != errorToken)
+                    tokensSinceLastError++;
+
+                if (tokensSinceLastError > 5)
+                    recovering = false;
+            }
+
+            if (NextToken != endOfFileToken)
+                NextToken = 0;
+        }
+
+        private void Reduce( int ruleNumber ) {
+#if TRACE_ACTIONS
+				DisplayRule(ruleNumber);
+#endif
+            Rule rule = rules[ruleNumber];
+            int rhLen = rule.RightHandSide.Length;
+            //
+            //  Default actions for unit productions.
+            //
+            if (rhLen == 1) {
+                CurrentSemanticValue = valueStack.TopElement();   // Default action: $$ = $1;
+                CurrentLocationSpan = LocationStack.TopElement(); // Default action "@$ = @1;
+            }
+            else if (rhLen == 0) {
+                // Create a new blank value.
+                // Explicit semantic action may mutate this value
+                CurrentSemanticValue = default( TValue );
+                // The location span for an empty production will start with the
+                // beginning of the next lexeme, and end with the finish of the
+                // previous lexeme.  This gives the correct behaviour when this
+                // nonsense value is used in later Merge operations.
+                CurrentLocationSpan = (scanner.yylloc != null && LastSpan != null ?
+                    scanner.yylloc.Merge( LastSpan ) :
+                    default( TSpan ));
+            }
+            else {
+                // Default action: $$ = $1;
+                CurrentSemanticValue = valueStack[LocationStack.Depth - rhLen];
+                //  Default action "@$ = @1.Merge(@N)" for location info.
+                TSpan at1 = LocationStack[LocationStack.Depth - rhLen];
+                TSpan atN = LocationStack[LocationStack.Depth - 1];
+                CurrentLocationSpan =
+                    ((at1 != null && atN != null) ? at1.Merge( atN ) : default( TSpan ));
+            }
+
+            DoAction( ruleNumber );
+
+            for (int i = 0; i < rule.RightHandSide.Length; i++) {
+                StateStack.Pop();
+                valueStack.Pop();
+                LocationStack.Pop();
+            }
+            FsaState = StateStack.TopElement();
+
+            if (FsaState.Goto.ContainsKey( rule.LeftHandSide ))
+                FsaState = states[FsaState.Goto[rule.LeftHandSide]];
+
+            StateStack.Push( FsaState );
+            valueStack.Push( CurrentSemanticValue );
+            LocationStack.Push( CurrentLocationSpan );
+        }
+
+        /// <summary>
+        /// Execute the selected action from array.
+        /// Must be overriden in derived classes.
+        /// </summary>
+        /// <param name="actionNumber">Index of the action to perform</param>
+        protected abstract void DoAction( int actionNumber );
+
+        private bool ErrorRecovery() {
+            bool discard;
+
+            if (!recovering) // if not recovering from previous error
+                ReportError();
+
+            if (!FindErrorRecoveryState())
+                return false;
+            //
+            //  The interim fix for the "looping in error recovery"
+            //  artifact involved moving the setting of the recovering 
+            //  bool until after invalid tokens have been discarded.
+            //
+            ShiftErrorToken();
+            discard = DiscardInvalidTokens();
+            recovering = true;
+            tokensSinceLastError = 0;
+            return discard;
+        }
+
+        private void ReportError() {
+            StringBuilder errorMsg = new StringBuilder();
+            errorMsg.AppendFormat( "Syntax error, unexpected {0}", TerminalToString( NextToken ) );
+
+            if (FsaState.ParserTable.Count < 7) {
+                bool first = true;
+                foreach (int terminal in FsaState.ParserTable.Keys) {
+                    if (first)
+                        errorMsg.Append( ", expecting " );
+                    else
+                        errorMsg.Append( ", or " );
+
+                    errorMsg.Append( TerminalToString( terminal ) );
+                    first = false;
+                }
+            }
+            scanner.yyerror( errorMsg.ToString() );
+        }
+
+        private void ShiftErrorToken() {
+            int old_next = NextToken;
+            NextToken = errorToken;
+
+            Shift( FsaState.ParserTable[NextToken] );
+
+#if TRACE_ACTIONS
+				Console.Error.WriteLine("Entering state {0} ", FsaState.number);
+#endif
+            NextToken = old_next;
+        }
+
+        private bool FindErrorRecoveryState() {
+            while (true)    // pop states until one found that accepts error token
+            {
+                if (FsaState.ParserTable != null &&
+                    FsaState.ParserTable.ContainsKey( errorToken ) &&
+                    FsaState.ParserTable[errorToken] > 0) // shift
+                    return true;
+
+#if TRACE_ACTIONS
+					Console.Error.WriteLine("Error: popping state {0}", StateStack.TopElement().number);
+#endif
+                StateStack.Pop();
+                valueStack.Pop();
+                LocationStack.Pop();
+
+#if TRACE_ACTIONS
+			    DisplayStack();
+#endif
+                if (StateStack.IsEmpty()) {
+#if TRACE_ACTIONS
+                        Console.Error.WriteLine("Aborting: didn't find a state that accepts error token");
+#endif
+                    return false;
+                }
+                else
+                    FsaState = StateStack.TopElement();
+            }
+        }
+
+        private bool DiscardInvalidTokens() {
+
+            int action = FsaState.defaultAction;
+
+            if (FsaState.ParserTable != null) {
+                // Discard tokens until find one that works ...
+                while (true) {
+                    if (NextToken == 0) {
+#if TRACE_ACTIONS
+                            Console.Error.Write("Reading a token: ");
+#endif
+                        NextToken = scanner.yylex();
+                    }
+#if TRACE_ACTIONS
+                        Console.Error.WriteLine("Next token is {0}", TerminalToString(NextToken));
+#endif
+                    if (NextToken == endOfFileToken)
+                        return false;
+
+                    if (FsaState.ParserTable.ContainsKey( NextToken ))
+                        action = FsaState.ParserTable[NextToken];
+
+                    if (action != 0)
+                        return true;
+                    else {
+#if TRACE_ACTIONS
+                            Console.Error.WriteLine("Error: Discarding {0}", TerminalToString(NextToken));
+#endif
+                        NextToken = 0;
+                    }
+                }
+            }
+            else if (recovering && tokensSinceLastError == 0) {
+                // 
+                //  Boolean recovering is not set until after the first
+                //  error token has been shifted.  Thus if we get back 
+                //  here with recovering set and no tokens read we are
+                //  looping on the same error recovery action.  This 
+                //  happens if current_state.ParserTable is null because
+                //  the state has an LR(0) reduction, but not all
+                //  lookahead tokens are valid.  This only occurs for
+                //  error productions that *end* on "error".
+                //
+                //  This action discards tokens one at a time until
+                //  the looping stops.  Another attack would be to always
+                //  use the LALR(1) table if a production ends on "error"
+                //
+#if TRACE_ACTIONS
+                    Console.Error.WriteLine("Error: panic discard of {0}", TerminalToString(NextToken));
+#endif
+                if (NextToken == endOfFileToken)
+                    return false;
+                NextToken = 0;
+                return true;
+            }
+            else
+                return true;
+        }
+
+        /// <summary>
+        /// Traditional YACC method.  Discards the next input token.
+        /// </summary>
+        [SuppressMessage( "Microsoft.Naming", "CA1709:IdentifiersShouldBeCasedCorrectly", MessageId = "yyclearin" )]
+        [SuppressMessage( "Microsoft.Naming", "CA1704:IdentifiersShouldBeSpelledCorrectly", MessageId = "yyclearin" )]
+        // Reason for FxCop message suppression -
+        // This is a traditional name for YACC-like functionality
+        protected void yyclearin() { NextToken = 0; }
+
+        /// <summary>
+        /// Tradional YACC method. Clear the "recovering" flag.
+        /// </summary>
+        [SuppressMessage( "Microsoft.Naming", "CA1709:IdentifiersShouldBeCasedCorrectly", MessageId = "yyerrok" )]
+        [SuppressMessage( "Microsoft.Naming", "CA1704:IdentifiersShouldBeSpelledCorrectly", MessageId = "yyerrok" )]
+        // Reason for FxCop message suppression -
+        // This is a traditional name for YACC-like functionality
+        protected void yyerrok() { recovering = false; }
+
+        /// <summary>
+        /// OBSOLETE FOR VERSION 1.4.0
+        /// Method used by derived types to insert new
+        /// state instances in the "states" array.
+        /// </summary>
+        /// <param name="stateNumber">index of the state</param>
+        /// <param name="state">data for the state</param>
+        protected void AddState( int stateNumber, State state ) {
+            states[stateNumber] = state;
+            state.number = stateNumber;
+        }
+
+        private void DisplayStack() {
+            Console.Error.Write( "State stack is now:" );
+            for (int i = 0; i < StateStack.Depth; i++)
+                Console.Error.Write( " {0}", StateStack[i].number );
+            Console.Error.WriteLine();
+        }
+
+        private void DisplayRule( int ruleNumber ) {
+            Console.Error.Write( "Reducing stack by rule {0}, ", ruleNumber );
+            DisplayProduction( rules[ruleNumber] );
+        }
+
+        private void DisplayProduction( Rule rule ) {
+            if (rule.RightHandSide.Length == 0)
+                Console.Error.Write( "/* empty */ " );
+            else
+                foreach (int symbol in rule.RightHandSide)
+                    Console.Error.Write( "{0} ", SymbolToString( symbol ) );
+
+            Console.Error.WriteLine( "-> {0}", SymbolToString( rule.LeftHandSide ) );
+        }
+
+        /// <summary>
+        /// Abstract state class naming terminal symbols.
+        /// This is overridden by derived classes with the
+        /// name (or alias) to be used in error messages.
+        /// </summary>
+        /// <param name="terminal">The terminal ordinal</param>
+        /// <returns></returns>
+        protected abstract string TerminalToString( int terminal );
+
+        private string SymbolToString( int symbol ) {
+            if (symbol < 0)
+                return nonTerminals[-symbol - 1];
+            else
+                return TerminalToString( symbol );
+        }
+
+        /// <summary>
+        /// Return text representation of argument character
+        /// </summary>
+        /// <param name="input">The character to convert</param>
+        /// <returns>String representation of the character</returns>
+        protected static string CharToString( char input ) {
+            switch (input) {
+                case '\a': return @"'\a'";
+                case '\b': return @"'\b'";
+                case '\f': return @"'\f'";
+                case '\n': return @"'\n'";
+                case '\r': return @"'\r'";
+                case '\t': return @"'\t'";
+                case '\v': return @"'\v'";
+                case '\0': return @"'\0'";
+                default: return string.Format( CultureInfo.InvariantCulture, "'{0}'", input );
+            }
+        }
+    }
+
+    /// <summary>
+    /// Classes implementing this interface must supply a
+    /// method that merges two location objects to return
+    /// a new object of the same type.
+    /// GPPG-generated parsers have the default location
+    /// action equivalent to "@$ = @1.Merge(@N);" where N
+    /// is the right-hand-side length of the production.
+    /// </summary>
+    /// <typeparam name="TSpan">The Location type</typeparam>
+#if EXPORT_GPPG
+    public interface IMerge<TSpan>
+#else
+    internal interface IMerge<TSpan>
+#endif
+ {
+        /// <summary>
+        /// Interface method that creates a location object from
+        /// the current and last object.  Typically used to create
+        /// a location object extending from the start of the @1
+        /// object to the end of the @N object.
+        /// </summary>
+        /// <param name="last">The lexically last object to merge</param>
+        /// <returns>The merged location object</returns>
+        TSpan Merge( TSpan last );
+    }
+
+    /// <summary>
+    /// This is the default class that carries location
+    /// information from the scanner to the parser.
+    /// If you don't declare "%YYLTYPE Foo" the parser
+    /// will expect to deal with this type.
+    /// </summary>
+#if EXPORT_GPPG
+    public class LexLocation : IMerge<LexLocation>
+#else
+    [SuppressMessage( "Microsoft.Performance", "CA1812:AvoidUninstantiatedInternalClasses" )]
+    internal class LexLocation : IMerge<LexLocation>
+#endif
+ {
+        private int startLine;   // start line
+        private int startColumn; // start column
+        private int endLine;     // end line
+        private int endColumn;   // end column
+
+        /// <summary>
+        /// The line at which the text span starts.
+        /// </summary>
+        public int StartLine { get { return startLine; } }
+
+        /// <summary>
+        /// The column at which the text span starts.
+        /// </summary>
+        public int StartColumn { get { return startColumn; } }
+
+        /// <summary>
+        /// The line on which the text span ends.
+        /// </summary>
+        public int EndLine { get { return endLine; } }
+
+        /// <summary>
+        /// The column of the first character
+        /// beyond the end of the text span.
+        /// </summary>
+        public int EndColumn { get { return endColumn; } }
+
+        /// <summary>
+        /// Default no-arg constructor.
+        /// </summary>
+        public LexLocation() { }
+
+        /// <summary>
+        /// Constructor for text-span with given start and end.
+        /// </summary>
+        /// <param name="sl">start line</param>
+        /// <param name="sc">start column</param>
+        /// <param name="el">end line </param>
+        /// <param name="ec">end column</param>
+        public LexLocation( int sl, int sc, int el, int ec ) { startLine = sl; startColumn = sc; endLine = el; endColumn = ec; }
+
+        /// <summary>
+        /// Create a text location which spans from the 
+        /// start of "this" to the end of the argument "last"
+        /// </summary>
+        /// <param name="last">The last location in the result span</param>
+        /// <returns>The merged span</returns>
+        public LexLocation Merge( LexLocation last ) { return new LexLocation( this.startLine, this.startColumn, last.endLine, last.endColumn ); }
+    }
+
+    /// <summary>
+    /// Abstract scanner class that GPPG expects its scanners to 
+    /// extend.
+    /// </summary>
+    /// <typeparam name="TValue">Semantic value type YYSTYPE</typeparam>
+    /// <typeparam name="TSpan">Source location type YYLTYPE</typeparam>
+#if EXPORT_GPPG
+    public abstract class AbstractScanner<TValue, TSpan>
+#else
+    internal abstract class AbstractScanner<TValue, TSpan>
+#endif
+ where TSpan : IMerge<TSpan> {
+        /// <summary>
+        /// Lexical value optionally set by the scanner. The value
+        /// is of the %YYSTYPE type declared in the parser spec.
+        /// </summary>
+        [SuppressMessage( "Microsoft.Design", "CA1051:DoNotDeclareVisibleInstanceFields" )]
+        [SuppressMessage( "Microsoft.Naming", "CA1709:IdentifiersShouldBeCasedCorrectly", MessageId = "yylval" )]
+        [SuppressMessage( "Microsoft.Naming", "CA1704:IdentifiersShouldBeSpelledCorrectly", MessageId = "yylval" )]
+        // Reason for FxCop message suppression -
+        // This is a traditional name for YACC-like functionality
+        // A field must be declared for this value of parametric type,
+        // since it may be instantiated by a value struct.  If it were 
+        // implemented as a property, machine generated code in derived
+        // types would not be able to select on the returned value.
+#pragma warning disable 649
+        public TValue yylval;                     // Lexical value: set by scanner
+#pragma warning restore 649
+
+        /// <summary>
+        /// Current scanner location property. The value is of the
+        /// type declared by %YYLTYPE in the parser specification.
+        /// </summary>
+        [SuppressMessage( "Microsoft.Naming", "CA1709:IdentifiersShouldBeCasedCorrectly", MessageId = "yylloc" )]
+        [SuppressMessage( "Microsoft.Naming", "CA1704:IdentifiersShouldBeSpelledCorrectly", MessageId = "yylloc" )]
+        // Reason for FxCop message suppression -
+        // This is a traditional name for YACC-like functionality
+        public virtual TSpan yylloc {
+            get { return default( TSpan ); }       // Empty implementation allowing
+            set { /* skip */ }                   // yylloc to be ignored entirely.
+        }
+
+        /// <summary>
+        /// Main call point for LEX-like scanners.  Returns an int
+        /// corresponding to the token recognized by the scanner.
+        /// </summary>
+        /// <returns>An int corresponding to the token</returns>
+        [SuppressMessage( "Microsoft.Naming", "CA1709:IdentifiersShouldBeCasedCorrectly", MessageId = "yylex" )]
+        [SuppressMessage( "Microsoft.Naming", "CA1704:IdentifiersShouldBeSpelledCorrectly", MessageId = "yylex" )]
+        // Reason for FxCop message suppression -
+        // This is a traditional name for YACC-like functionality
+        public abstract int yylex();
+
+        /// <summary>
+        /// Traditional error reporting provided by LEX-like scanners
+        /// to their YACC-like clients.
+        /// </summary>
+        /// <param name="format">Message format string</param>
+        /// <param name="args">Optional array of args</param>
+        [SuppressMessage( "Microsoft.Naming", "CA1709:IdentifiersShouldBeCasedCorrectly", MessageId = "yyerror" )]
+        [SuppressMessage( "Microsoft.Naming", "CA1704:IdentifiersShouldBeSpelledCorrectly", MessageId = "yyerror" )]
+        // Reason for FxCop message suppression -
+        // This is a traditional name for YACC-like functionality
+        public virtual void yyerror( string format, params object[] args ) { }
+    }
+
+    /// <summary>
+    /// Encapsulated state for the parser.
+    /// Opaque to users, visible to the tool-generated code.
+    /// </summary>
+#if EXPORT_GPPG
+    public class State
+    {
+        /// <summary>
+        /// The number of states in the automaton.
+        /// </summary>
+        public int number;
+#else
+    internal class State {
+        /// <summary>
+        /// The index of this state in the states array.
+        /// </summary>
+        internal int number;
+#endif
+        internal Dictionary<int, int> ParserTable;   // Terminal -> ParseAction
+        internal Dictionary<int, int> Goto;          // NonTerminal -> State;
+        internal int defaultAction; // = 0;		     // ParseAction
+
+        /// <summary>
+        /// State transition data for this state. Pairs of elements of the 
+        /// goto array associate symbol ordinals with next state indices.
+        /// The actions array is passed to another constructor. 
+        /// </summary>
+        /// <param name="actions">The action list</param>c
+        /// <param name="goToList">Next state data</param>
+        public State( int[] actions, int[] goToList )
+            : this( actions ) {
+            Goto = new Dictionary<int, int>();
+            for (int i = 0; i < goToList.Length; i += 2)
+                Goto.Add( goToList[i], goToList[i + 1] );
+        }
+
+        /// <summary>
+        /// Action data for this state. Pairs of elements of the 
+        /// action array associate action ordinals with each of
+        /// those symbols that have actions in the current state.
+        /// </summary>
+        /// <param name="actions">The action array</param>
+        public State( int[] actions ) {
+            ParserTable = new Dictionary<int, int>();
+            for (int i = 0; i < actions.Length; i += 2)
+                ParserTable.Add( actions[i], actions[i + 1] );
+        }
+
+        /// <summary>
+        /// Set the default action for this state.
+        /// </summary>
+        /// <param name="defaultAction">Ordinal of the default action</param>
+        public State( int defaultAction ) {
+            this.defaultAction = defaultAction;
+        }
+
+        /// <summary>
+        /// Set the default action and the state transition table.
+        /// </summary>
+        /// <param name="defaultAction">The default action</param>
+        /// <param name="goToList">Transitions from this state</param>
+        public State( int defaultAction, int[] goToList )
+            : this( defaultAction ) {
+            Goto = new Dictionary<int, int>();
+            for (int i = 0; i < goToList.Length; i += 2)
+                Goto.Add( goToList[i], goToList[i + 1] );
+        }
+    }
+
+    /// <summary>
+    /// Rule representation at runtime.
+    /// </summary>
+#if EXPORT_GPPG
+    public class Rule
+#else
+    internal class Rule
+#endif
+ {
+        internal int LeftHandSide; // symbol
+        internal int[] RightHandSide; // symbols
+
+        /// <summary>
+        /// Rule constructor.  This holds the ordinal of
+        /// the left hand side symbol, and the list of
+        /// right hand side symbols, in lexical order.
+        /// </summary>
+        /// <param name="left">The LHS non-terminal</param>
+        /// <param name="right">The RHS symbols, in lexical order</param>
+        public Rule( int left, int[] right ) {
+            this.LeftHandSide = left;
+            this.RightHandSide = right;
+        }
+    }
+
+    /// <summary>
+    /// Stack utility for the shift-reduce parser.
+    /// GPPG parsers have three instances:
+    /// (1) The parser state stack, T = QUT.Gppg.State,
+    /// (2) The semantic value stack, T = TValue,
+    /// (3) The location stack, T = TSpan.
+    /// </summary>
+    /// <typeparam name="T"></typeparam>
+#if EXPORT_GPPG
+    public class PushdownPrefixState<T>
+#else
+    internal class PushdownPrefixState<T>
+#endif
+ {
+        //  Note that we cannot use the BCL Stack<T> class
+        //  here as derived types need to index into stacks.
+        //
+        private T[] array = new T[8];
+        private int tos = 0;
+
+        /// <summary>
+        /// Indexer for values of the stack below the top.
+        /// </summary>
+        /// <param name="index">index of the element, starting from the bottom</param>
+        /// <returns>the selected element</returns>
+        public T this[int index] { get { return array[index]; } }
+
+        /// <summary>
+        /// The current depth of the stack.
+        /// </summary>
+        public int Depth { get { return tos; } }
+
+        internal void Push( T value ) {
+            if (tos >= array.Length) {
+                T[] newarray = new T[array.Length * 2];
+                System.Array.Copy( array, newarray, tos );
+                array = newarray;
+            }
+            array[tos++] = value;
+        }
+
+        internal T Pop() {
+            T rslt = array[--tos];
+            array[tos] = default( T );
+            return rslt;
+        }
+
+        internal T TopElement() { return array[tos - 1]; }
+
+        internal bool IsEmpty() { return tos == 0; }
+    }
+}
+

src/DataStreamSuite/Bts.ColumnSortExpressions/Usings.cs

@@ -0,0 +1,4 @@
+global using System;
+global using System.Linq;
+global using System.Collections.Generic;
+global using System.Collections.Immutable;

src/DataStreamSuite/ColumnMapExpressions/ColumnMapExpressions.csproj

@@ -8,7 +8,7 @@
 
   <ItemGroup>
     <PackageReference Include="Google.Protobuf" Version="3.24.3" />
-    <PackageReference Include="YaccLexTools" Version="1.2.0" />
+    <PackageReference Include="YaccLexTools" Version="1.2.2" />
   </ItemGroup>
 
   <ItemGroup>

src/DataStreamSuite/DataStream/DataStream.csproj

@@ -9,6 +9,8 @@
     <ProjectReference Include="..\ArithmeticExpressions\ArithmeticExpressions.csproj" />
     <ProjectReference Include="..\BaseUtilities\BaseUtilities.csproj" />    
     <ProjectReference Include="..\BooleanExpressions\BooleanExpressions.csproj" />
+    <ProjectReference Include="..\Bts.ColumnExpressions\Bts.ColumnExpressions.csproj" />
+    <ProjectReference Include="..\Bts.ColumnSortExpressions\Bts.ColumnSortExpressions.csproj" />
     <ProjectReference Include="..\DataStreamCore\DataStreamCore.csproj" />
     <ProjectReference Include="..\DataStreamExpressions\DataStreamExpressions.csproj" />
     <ProjectReference Include="..\..\client\spark_core\spark_core.csproj" />

src/DataStreamSuite/DataStream/Misc/PandasStream.cs

@@ -0,0 +1,62 @@
+// using System;
+// using System.Diagnostics;
+
+// using BCG;
+// using BCG.Utilities.General;
+
+
+// namespace bts {
+
+//     public class PandasStream : ViewStream, IWorker {
+//         //C:\Users\Owner\AppData\Local\Microsoft\WindowsApps\python.exe
+
+//         public PandasStream(ViewStream vs, ViewStreamDatabase vsd, string pandas_code) {
+
+//         }
+
+//         public PandasStream(ViewStream vs, string pandas_code) :
+//         this(vs, ViewStreamDatabase.instance, pandas_code) {
+//         }
+
+
+//         protected override void myOnFirstSubscriber() {
+
+//             // string python_code = @"
+//             //    import winsound
+//             //    frequency = 2500  # Set frequency (in Hz)
+//             //    duration = 1000   # Set duration (in milliseconds)
+//             //    winsound.Beep(frequency, duration)
+//             // ";
+//             string python_code = @"
+//             file_path = \"C:\\Temp\\test2.txt\\"  # r prefix for raw string to avoid escape characters
+//             with open(file_path, \"w\") as f:
+//                 f.write(\"Hello, world!\n\")
+//             ";
+
+//             execute_python_code(python_code);
+//         }
+
+//         static void execute_python_code(string python_code) {
+//             ProcessStartInfo start = new ProcessStartInfo();
+//             start.FileName = "python"; // Python executable
+//             start.Arguments = "-c \"" + python_code.Replace("\"", "\\\"") + "\""; // Pass code as argument
+//             start.UseShellExecute = false;
+//             start.RedirectStandardOutput = true;
+
+//             using (Process process = Process.Start(start)) {
+//                 using (System.IO.StreamReader reader = process.StandardOutput) {
+//                     string result = reader.ReadToEnd();
+//                     Console.Write(result);
+//                 }
+//             }
+//         }
+
+//         protected override void myOnLastSubscriber() {
+//         }
+
+//         public void doWork() {
+
+//         }
+
+//     }
+// }

src/DataStreamSuite/DataStream/Misc/TCPConnectionTestStream.cs

@@ -0,0 +1,79 @@
+using BCG.Utilities.General;
+
+using System;
+
+namespace BCG {
+    public class TCPConnectionTestStream : ViewStream, IWorker, ITCPConnectionHandler, ITCPMessageHandler {
+        class State {
+            public int n_sends = 0;
+            public int n_recvs = 0;
+        }
+        private readonly string _ip;
+        private readonly int _port;
+        private readonly ViewStreamDatabase _vsd;
+        State _state = new State();
+        private readonly int _msg_size;
+
+        private IBlueTableSchema _schema;
+        private TimeSpan _send_period;
+        private ITCPConnection _connection = null;
+        private byte[] _buff;
+
+        public TCPConnectionTestStream(string ip, int port, int msg_size, int send_period_seconds) :
+        this(ip, port, msg_size, send_period_seconds, ViewStreamDatabase.instance) { }
+        public TCPConnectionTestStream(string ip, int port, int msg_size, int send_period_seconds, ViewStreamDatabase vsd) {
+            _ip = ip;
+            _port = port;
+            _msg_size = msg_size;
+            _vsd = vsd;
+            _schema = BlueDataUtility.form_schema("TCPClient",
+            Tuple.Create("id", TypeCode.Int32),
+            Tuple.Create("n_sends", TypeCode.UInt32),
+            Tuple.Create("n_recvs", TypeCode.UInt32));
+            _send_period = TimeSpan.FromSeconds(send_period_seconds);
+            _buff = new byte[msg_size];
+            for (int i = 0; i < _buff.Length; i++) {
+                _buff[i] = (byte)i;
+            }
+        }
+
+        protected override void myOnFirstSubscriber() {
+            MasterWorker.getInstance(false).addWorker(this, _send_period);
+            var server_address = new BlutoSubAddress(_ip, _port);
+            bts.AsyncTCPConnector_APM.connect_to(server_address, this);
+        }
+
+        protected override void myOnLastSubscriber() {
+            MasterWorker.getInstance(false).removeWorker(this);
+        }
+
+        public void doWork() {
+            if (_connection != null) {
+                _connection.send(_buff);
+                _state.n_sends++;
+            }
+            IBlueRow r = new BlueRow(_schema, 0, _state.n_sends, _state.n_recvs);
+            BlueTable rows = new BlueTable(r.schema);
+            rows.addRow(r);
+            var dp = BlueDataUtility.build_dp_inc(rows, myLastValue as DataPacket, true);
+            myFireEvent(dp);
+        }
+
+        public void onConnection(ITCPConnection connection) {
+            _connection = connection;
+            _connection.register(this);
+        }
+        public bool isDisposed { get; }
+
+        public void onData(RemoteMessage msg) {
+            bts.Log.Gen.info($"TCPConnectionTestStream received msg: {msg}");
+            _state.n_recvs++;
+        }
+
+        public void Dispose() {
+
+        }
+
+
+    }
+}

src/DataStreamSuite/DataStream/Misc/test.py

@@ -0,0 +1,3 @@
+file_path = r"C:\Temp\test2.txt"  # r prefix for raw string to avoid escape characters
+with open(file_path, "w") as f:
+    f.write("Hello, world!\n")

src/DataStreamSuite/DataStream/Primitives/AddColumns/AddColumnsStream.cs

@@ -1,11 +1,6 @@
-// using System;
-// using System.Data;
-// using System.Collections;
-// using BCG.Utilities;
-// using BCG.Utilities.General;
-// namespace BCG {
-
-//     public class AddColumnsStream : MapStream {
-//         public AddColumnsStream(ViewStream vs, string columnDef) : base(vs, new ColumnAdder(columnDef)) { }
-//     }
-// }
+namespace BCG;
+
+public class AddColumnsStream : MapStream {
+   public AddColumnsStream(ViewStream vs, string columnDef)
+      : base(vs, new ColumnAdder(columnDef)) { }
+}

src/DataStreamSuite/DataStream/Primitives/AddColumns/AddConstantColumnRow.cs

@@ -0,0 +1,12 @@
+namespace bts {
+    using BCG;
+
+    using System;
+    using System.Collections;
+
+    public class AddConstantColumnRow {
+        AddConstantColumnRow(IBlueRow r, Tuple<string, Type, object>[] constant_cols) {
+
+        }
+    }
+}

src/DataStreamSuite/DataStream/Primitives/AddColumns/ColumnAdder.cs

@@ -1,63 +1,52 @@
-// using System;
-// using System.Collections;
-// using BCG.Utilities;
-// using BCG.Evaluator.AddColumnsExpression;
-
-// namespace BCG {
-//     /// <summary>
-//     /// Provides methods to add columns to the schema
-//     /// </summary>
-//     public class ColumnAdder : IStreamMapper {
-//         protected IDictionary myNewColumns;
-//         protected IBlueTableSchema mySchema;
-
-//         static syntax myParser =
-//            new syntax();
-
-//         /// <summary>
-//         /// string represting the columns to be added to the output table. 
-//         /// </summary>
-//         /// <param name="colAddExpression"></param>
-//         public ColumnAdder(string columnDef) {
-//             AddColumnsExpression ace;
-
-//             if (DataStreamLoadFileConfig.instance.useStaticParser) {
-//                 lock (myParser) {
-//                     ace =
-//                     (AddColumnsExpression)myParser.Parse(columnDef);
-//                 }
-//             } else {
-//                 syntax parser = new syntax();
-//                 ace =
-//                 (AddColumnsExpression)parser.Parse(columnDef);
-//             }
-
-//             if (ace == null)
-//                 throw new Exception("Invalid columnn expression: " + columnDef);
-//             myNewColumns = ace.columns;
-//         }
-
-//         public virtual IBlueRow map(IBlueRow row) {
-//             if (mySchema == null)
-//                 myConstructSchema(row.schema);
-//             return new AddColumnsRow(mySchema, row);
-//         }
-
-//         public IBlueTableSchema getOutputSchema(IBlueTableSchema inputSchema) {
-//             myConstructSchema(inputSchema);
-//             return mySchema;
-//         }
-
-//         private void myConstructSchema(IBlueTableSchema schema) {
-//             ColumnInfo[] cis = new ColumnInfo[schema.columnNames.Length + myNewColumns.Count];
-//             int ciIndex = 0;
-//             foreach (ColumnInfo ci in schema.columnInfo)
-//                 cis[ciIndex++] = ci;
-//             foreach (DictionaryEntry de in myNewColumns)
-//                 cis[ciIndex] = new ColumnInfo((string)de.Key, ciIndex++,
-//                    ((AddColumnsExpression.ConstantInfo)de.Value).t, ((AddColumnsExpression.ConstantInfo)de.Value).v);
-//             mySchema = new BlueTableSchema(cis, schema.tableName);
-//         }
-
-//     }
-// }
+using System;
+using System.Collections.Generic;
+using System.Diagnostics.CodeAnalysis;
+using System.Linq;
+
+using Bts.ColumnExpressions;
+
+namespace BCG;
+
+/// <summary>
+/// Provides methods to add columns to the schema
+/// </summary>
+/// <param name="columnsExpression">
+/// String represting the columns to add to the output table
+/// </param>
+public class ColumnAdder : IStreamMapper {
+   // Store tuples of (column, index) to preserve order
+   protected readonly IDictionary<string, (ColumnExpression column, int index)> myNewColumns;
+   protected IBlueTableSchema? mySchema;
+
+   public ColumnAdder(string columnsExpression) =>
+      myNewColumns = (ColumnExpressionInterpreter.Evaluate(columnsExpression) ??
+         throw new Exception("Invalid columnn expression: " + columnsExpression))
+         .Select((c, i) => (c, i))
+         .ToDictionary(c => c.c.Name, c => (c.c, c.i));
+
+   [MemberNotNull(nameof(mySchema))]
+   public virtual IBlueRow map(IBlueRow row) {
+      // Only rebuild schema if necessary (i.e., if it's null)
+      mySchema ??= buildSchema(row.schema, myNewColumns);
+      return new AddColumnsRow(mySchema, row);
+   }
+
+   [MemberNotNull(nameof(mySchema))]
+   public IBlueTableSchema getOutputSchema(IBlueTableSchema inputSchema) =>
+      // Always rebuild schema (even if it's already non-null)
+      mySchema = buildSchema(inputSchema, myNewColumns);
+
+   [MemberNotNull(nameof(mySchema))]
+   static BlueTableSchema buildSchema(
+      IBlueTableSchema schema,
+      IDictionary<string, (ColumnExpression column, int index)> newColumns
+   ) {
+      ColumnInfo[] cis = new ColumnInfo[schema.columnNames.Length + newColumns.Count];
+      schema.columnInfo.CopyTo(cis, 0);
+      foreach (var (col, index) in newColumns.Values) {
+         var schemaIdx = schema.columnInfo.Length + index;
+         cis[schemaIdx] = new ColumnInfo(col.Name, schemaIdx, col.Type, col.Value);
+      }
+      return new BlueTableSchema(cis, schema.tableName);
+   }
+}

src/DataStreamSuite/DataStream/Primitives/FileStreams/PersistorStream.cs

@@ -15,6 +15,11 @@ namespace BCG {
    // data:
    // epochtime, size, serialized data
    [DataStreamUserRespresentation("pers")]
+   // Saves each udpate of a stream into a file. The token is the the unique logical name of the stream,
+   // which maps the name of the directory where the streams are saved. The directory (3rd arg) is the 
+   // parent directory where the token folder is stored. Each file goes up to rolloevr_size after which a new
+   // file is created. The file names are token_epoch_datetime
+   // the first entry in a file is the complete state
    public class PersistorStream : ViewStream {
       private ViewStream _source_stream;
       private string _token;

src/DataStreamSuite/DataStream/Primitives/FileStreams/SnapshotStream.cs

@@ -11,13 +11,27 @@ using System.Reflection.Metadata;
 
 namespace BCG {
    public class SnapshotStream : ViewStream {
-
+      public enum SpecificSnapshot {
+         FirstOfDay,
+         LastOfDay
+      };
       private BlueTable _current_table;
       private IClientDeserializer _deserilializer = new ClientUniversalDeserializer();
       public DateTime _snapshot_time;
       public string _snapshot_file;
       public IEnumerable<PlaybackPacket> _packets;
-
+      public SnapshotStream(string snapshot_date, string specific_snapshot, string directory_name, string token) {
+         DateTime dt;
+         var parsed = DateTime.TryParse(snapshot_date, out dt);
+         dt = dt.Date;
+         bts.Log.assert(parsed);
+         SpecificSnapshot specific_snapshot_enum;
+         parsed = Enum.TryParse<SpecificSnapshot>(specific_snapshot, out specific_snapshot_enum);
+         bts.Log.assert(parsed);
+         string snapshot_dt = StreamPersistorUtils.find_snapshot_file($"{directory_name}\\{token}", dt, specific_snapshot_enum);
+         ViewStream rs = new SnapshotStream(snapshot_dt, directory_name, token);
+         mySetRealSource(rs);
+      }
 
       public SnapshotStream(string snapshot_dt, string directory_name, string token) {
          _snapshot_time = PersistorStream.parse_datetime(snapshot_dt);

src/DataStreamSuite/DataStream/Primitives/FileStreams/StreamPersistorUtils.cs

@@ -37,6 +37,20 @@ namespace BCG {
       public IEnumerable<PlaybackPacket> packets = new List<PlaybackPacket>();
    }
    public static class StreamPersistorUtils {
+      public static string find_snapshot_file(string directory_name, DateTime snapshot_date, SnapshotStream.SpecificSnapshot specific_snapshot) {
+         DateTime max_dt = DateTime.MinValue;
+         string rv = String.Empty;
+         string[] file_names = Directory.GetFiles(directory_name);
+         for (int i = 0; i < file_names.Count() - 1; i++) {
+            string fn = file_names[i];
+            var dt = parse_datetime_from_filename(fn);
+            if (dt > max_dt) {
+               max_dt = dt;
+               rv = fn;
+            }
+         }
+         return rv;
+      }
       public static string find_snapshot_file(string directory_name, DateTime snapshot_time) {
          string[] file_names = Directory.GetFiles(directory_name);
          for (int i = 0; i < file_names.Count() - 1; i++) {

src/DataStreamSuite/DataStream/Primitives/GroupBy/MaxGroupByEvaluator.cs

@@ -1,47 +1,42 @@
-// using System;
-// using System.Collections;
-// using BCG.Utilities;
+using System;
+using System.Collections;
 
-// namespace BCG
-// {
-// 	/// <summary>
-// 	/// Computes the max for a given column
-// 	/// </summary>
-// 	public class MaxGroupByEvaluator : GroupByEvaluator
-// 	{
+using BCG.Utilities;
 
-//       public MaxGroupByEvaluator (string outputCol, string[] groupByCols) : base (outputCol, groupByCols)
-//       {}
+namespace BCG {
+    /// <summary>
+    /// Computes the max for a given column
+    /// </summary>
+    public class MaxGroupByEvaluator : GroupByEvaluator {
 
-//       public MaxGroupByEvaluator (string colMapEpression, string outputCol, string[] groupByCols)
-//          : base (colMapEpression, outputCol, groupByCols)   {}
+        public MaxGroupByEvaluator(string outputCol, string[] groupByCols) : base(outputCol, groupByCols) { }
 
-//       public override IBlueRow evaluate (DataPacket dp)  
-//       {
-//          myOutputTable = myOutputTable == null ? new GroupByOutputTable (dp.schema) :
-//             myOutputTable;
-//          string hashString = BlueDataUtility.rowToString (GroupByStream.getRepRow (dp), myGroupByCols);
-//          IBlueRow currMaxRow = myOutputTable.getRow (hashString);
-//          IBlueRow maxRow = BlueDataUtility.computeMaxRow (dp.deletedRows, myOutputColumn);
-//          if (maxRow != null)
-//          {
-//             BlueEventLog.logAssert (currMaxRow != null);
-//             BlueEventLog.logAssert (BlueDataUtility.compareRows (maxRow, currMaxRow, myOutputColumn) <= 0);
-//             if (BlueDataUtility.compareRows (maxRow, currMaxRow, myOutputColumn) == 0)
-//             {
-//                maxRow = BlueDataUtility.computeMaxRow (dp.newRows, myOutputColumn);
-//                if (maxRow == null)
-//                   myOutputTable.removeRow (hashString);
-//                else
-//                   myOutputTable.addRow (maxRow, hashString);
-//             }
-//          }
-//          maxRow = BlueDataUtility.computeMaxRow (dp.addedRows, myOutputColumn);
-//          // if the row was replaced in the above lines, then currMaxRow < maxRow, so the if below would never return a true,
-//          // otherwise if maxRow > currMaxRow then its replaced
-//          if (maxRow != null && (currMaxRow == null || BlueDataUtility.compareRows (maxRow, currMaxRow, myOutputColumn) > 0))
-//             myOutputTable.addRow (maxRow, hashString);
-//          return myProject (myOutputTable.getRow (hashString));
-//       }
-// 	}
-// }
+        public MaxGroupByEvaluator(string colMapEpression, string outputCol, string[] groupByCols)
+           : base(colMapEpression, outputCol, groupByCols) { }
+
+        public override IBlueRow evaluate(DataPacket dp) {
+            myOutputTable = myOutputTable == null ? new GroupByOutputTable(dp.schema) :
+               myOutputTable;
+            string hashString = BlueDataUtility.rowToString(GroupByStream.getRepRow(dp), myGroupByCols);
+            IBlueRow currMaxRow = myOutputTable.getRow(hashString);
+            IBlueRow maxRow = BlueDataUtility.computeMaxRow(dp.deletedRows, myOutputColumn);
+            if (maxRow != null) {
+                bts.Log.assert(currMaxRow != null);
+                bts.Log.assert(BlueDataUtility.compareRows(maxRow, currMaxRow, myOutputColumn) <= 0);
+                if (BlueDataUtility.compareRows(maxRow, currMaxRow, myOutputColumn) == 0) {
+                    maxRow = BlueDataUtility.computeMaxRow(dp.newRows, myOutputColumn);
+                    if (maxRow == null)
+                        myOutputTable.removeRow(hashString);
+                    else
+                        myOutputTable.addRow(maxRow, hashString);
+                }
+            }
+            maxRow = BlueDataUtility.computeMaxRow(dp.addedRows, myOutputColumn);
+            // if the row was replaced in the above lines, then currMaxRow < maxRow, so the if below would never return a true,
+            // otherwise if maxRow > currMaxRow then its replaced
+            if (maxRow != null && (currMaxRow == null || BlueDataUtility.compareRows(maxRow, currMaxRow, myOutputColumn) > 0))
+                myOutputTable.addRow(maxRow, hashString);
+            return myProject(myOutputTable.getRow(hashString));
+        }
+    }
+}

src/DataStreamSuite/DataStream/Primitives/GroupBy/MaxStream.cs

@@ -1,20 +1,15 @@
-// using System;
+using System;
 
-// namespace BCG
-// {
-// 	public class MaxStream : ViewStream
-// 	{
-//       public MaxStream(ViewStream stream, string outputCol, string inputCols) : this(stream, outputCol, inputCols.Split(' '))
-//       {
-//       }
-// 		public MaxStream (ViewStream stream, string outputCol, string[] inputCols)
-// 		{
-//          mySetRealSource (new GroupByStream (stream, 
-//                                              new MaxGroupByEvaluator (outputCol, inputCols)));
-// 		}
+namespace BCG {
+    public class MaxStream : ViewStream {
+        public MaxStream(ViewStream stream, string outputCol, string inputCols) : this(stream, outputCol, inputCols.Split(' ')) {
+        }
+        public MaxStream(ViewStream stream, string outputCol, string[] inputCols) {
+            mySetRealSource(new GroupByStream(stream,
+                                                new MaxGroupByEvaluator(outputCol, inputCols)));
+        }
 
-//       public MaxStream (ViewStream stream, string outputCol)
-//          : this (new AddColumnsStream (stream, "fakeCol=0"), outputCol, new string[]{"fakeCol"})
-//       {}
-// 	}
-// }
+        public MaxStream(ViewStream stream, string outputCol)
+           : this(new AddColumnsStream(stream, "fakeCol=0"), outputCol, new string[] { "fakeCol" }) { }
+    }
+}

src/DataStreamSuite/DataStream/Primitives/GroupBy/MinGroupByEvaluator.cs

@@ -1,40 +1,36 @@
-// using System;
-// using BCG.Utilities;
+using System;
 
-// namespace BCG
-// {
-//    public class MinGroupByEvaluator : MaxGroupByEvaluator
-//    {
-//       public MinGroupByEvaluator (string outputCol, string[] groupByCols) 
-//          : base (outputCol, groupByCols)  {}
+using BCG.Utilities;
 
-//       public MinGroupByEvaluator (string colMapEpression, string outputCol, string[] groupByCols) :
-//          base (colMapEpression, outputCol, groupByCols)  {}
+namespace BCG {
+    public class MinGroupByEvaluator : MaxGroupByEvaluator {
+        public MinGroupByEvaluator(string outputCol, string[] groupByCols)
+           : base(outputCol, groupByCols) { }
 
-//       public override IBlueRow evaluate (DataPacket dp)  
-//       {
-//          myOutputTable = myOutputTable == null ? new GroupByOutputTable (dp.schema) : myOutputTable;
-//          string stringRep = BlueDataUtility.rowToString (GroupByStream.getRepRow (dp), myGroupByCols);
-//          IBlueRow currMinRow = myOutputTable.getRow (stringRep);
-//          IBlueRow minRow = BlueDataUtility.computeMinRow (dp.deletedRows, myOutputColumn);
-//          if (minRow != null)
-//          {
-//             // BlueEventLog.logAssert (currMinRow != null);
-//             // BlueEventLog.logAssert (BlueDataUtility.compareRows (minRow, currMinRow, myOutputColumn) >= 0);
-//             if (BlueDataUtility.compareRows (minRow, currMinRow, myOutputColumn) == 0)
-//             {
-//                minRow = BlueDataUtility.computeMinRow (dp.newRows, myOutputColumn);
-//                if (minRow == null)
-//                   myOutputTable.removeRow (stringRep);
-//                else
-//                   myOutputTable.addRow (minRow, stringRep);
-//             }
-//          }
-//          minRow = BlueDataUtility.computeMinRow (dp.addedRows, myOutputColumn);
-//          if (minRow != null && (currMinRow == null || BlueDataUtility.compareRows (minRow, currMinRow, myOutputColumn) < 0))
-//             myOutputTable.addRow (minRow, stringRep);
-//          return myOutputTable.getRow (stringRep);
-//       }
+        public MinGroupByEvaluator(string colMapEpression, string outputCol, string[] groupByCols) :
+           base(colMapEpression, outputCol, groupByCols) { }
 
-//    }
-// }
+        public override IBlueRow evaluate(DataPacket dp) {
+            myOutputTable = myOutputTable == null ? new GroupByOutputTable(dp.schema) : myOutputTable;
+            string stringRep = BlueDataUtility.rowToString(GroupByStream.getRepRow(dp), myGroupByCols);
+            IBlueRow currMinRow = myOutputTable.getRow(stringRep);
+            IBlueRow minRow = BlueDataUtility.computeMinRow(dp.deletedRows, myOutputColumn);
+            if (minRow != null) {
+                // BlueEventLog.logAssert (currMinRow != null);
+                // BlueEventLog.logAssert (BlueDataUtility.compareRows (minRow, currMinRow, myOutputColumn) >= 0);
+                if (BlueDataUtility.compareRows(minRow, currMinRow, myOutputColumn) == 0) {
+                    minRow = BlueDataUtility.computeMinRow(dp.newRows, myOutputColumn);
+                    if (minRow == null)
+                        myOutputTable.removeRow(stringRep);
+                    else
+                        myOutputTable.addRow(minRow, stringRep);
+                }
+            }
+            minRow = BlueDataUtility.computeMinRow(dp.addedRows, myOutputColumn);
+            if (minRow != null && (currMinRow == null || BlueDataUtility.compareRows(minRow, currMinRow, myOutputColumn) < 0))
+                myOutputTable.addRow(minRow, stringRep);
+            return myOutputTable.getRow(stringRep);
+        }
+
+    }
+}

src/DataStreamSuite/DataStream/Primitives/GroupBy/MinStream.cs

@@ -1,17 +1,13 @@
-// using System;
+using System;
 
-// namespace BCG
-// {
-//    public class MinStream : ViewStream
-//    {
-//       public MinStream (ViewStream stream, string outputCol, string[] inputCol)
-//       {
-//          mySetRealSource (new GroupByStream (stream, 
-//                            new MinGroupByEvaluator (outputCol, inputCol)));
-//       }
+namespace BCG {
+    public class MinStream : ViewStream {
+        public MinStream(ViewStream stream, string outputCol, string[] inputCol) {
+            mySetRealSource(new GroupByStream(stream,
+                              new MinGroupByEvaluator(outputCol, inputCol)));
+        }
 
-//       public MinStream (ViewStream stream, string outputCol)
-//                      : this (new AddColumnsStream (stream, "fakeCol=0"), outputCol, new string[]{"fakeCol"})
-//       {}
-//    }
-// }
+        public MinStream(ViewStream stream, string outputCol)
+                       : this(new AddColumnsStream(stream, "fakeCol=0"), outputCol, new string[] { "fakeCol" }) { }
+    }
+}

src/DataStreamSuite/DataStream/Primitives/GroupBy/SumGroupByEvaluator.cs

@@ -1,94 +1,82 @@
-// using System;
+using System;
 
-// namespace BCG
-// {
-//    /// <summary>
-//    /// Computes the sum for a given column
-//    /// </summary>
-//    public class SumGroupByEvaluator : GroupByEvaluator
-//    {
-//       public SumGroupByEvaluator (string outputCol, string[] groupByCols) : base (outputCol, groupByCols)
-//       {}
+namespace BCG {
+    /// <summary>
+    /// Computes the sum for a given column
+    /// </summary>
+    public class SumGroupByEvaluator : GroupByEvaluator {
+        public SumGroupByEvaluator(string outputCol, string[] groupByCols) : base(outputCol, groupByCols) { }
 
-//       public SumGroupByEvaluator (string colMapEpression, string outputCol, string[] groupByCols)
-//          : base (colMapEpression, outputCol, groupByCols)   {}
+        public SumGroupByEvaluator(string colMapEpression, string outputCol, string[] groupByCols)
+           : base(colMapEpression, outputCol, groupByCols) { }
 
-//       public override IBlueRow evaluate (DataPacket dp)  
-//       {
-//          myOutputTable = myOutputTable == null ? new GroupByOutputTable (dp.schema) : myOutputTable;
-         
-//          IBlueRow sumRow = BlueDataUtility.computeSum(dp.newRows, myOutputColumn);
-//          return sumRow;
-//       }
-//    }
+        public override IBlueRow evaluate(DataPacket dp) {
+            myOutputTable = myOutputTable == null ? new GroupByOutputTable(dp.schema) : myOutputTable;
+
+            IBlueRow sumRow = BlueDataUtility.computeSum(dp.newRows, myOutputColumn);
+            return sumRow;
+        }
+    }
 
 
-//    public class MultiColumnSumGroupByEvaluator : IGroupByEvaluator
-//    {
-//       private string[] myOutputCols, myGroupByCols;
-//       public MultiColumnSumGroupByEvaluator(string spaceSeperatedOutputCols, string spaceSeperatedGroupByCols)
-//       {
-//          myOutputCols = spaceSeperatedOutputCols.Split(' ');
-//          myGroupByCols = spaceSeperatedGroupByCols.Split(' ');
-//       }
+    public class MultiColumnSumGroupByEvaluator : IGroupByEvaluator {
+        private string[] myOutputCols, myGroupByCols;
+        public MultiColumnSumGroupByEvaluator(string spaceSeperatedOutputCols, string spaceSeperatedGroupByCols) {
+            myOutputCols = spaceSeperatedOutputCols.Split(' ');
+            myGroupByCols = spaceSeperatedGroupByCols.Split(' ');
+        }
 
-//       public MultiColumnSumGroupByEvaluator(string[] output_columns, string[] groupby_columns)
-//       {
-//          myOutputCols = output_columns;
-//          myGroupByCols = groupby_columns;
-//       }
+        public MultiColumnSumGroupByEvaluator(string[] output_columns, string[] groupby_columns) {
+            myOutputCols = output_columns;
+            myGroupByCols = groupby_columns;
+        }
 
-//       #region IGroupByEvaluator Members
+        #region IGroupByEvaluator Members
 
-//       public IBlueRow evaluate(DataPacket packet)  {return BlueDataUtility.computeSum(packet.newRows, myOutputCols);}
+        public IBlueRow evaluate(DataPacket packet) { return BlueDataUtility.computeSum(packet.newRows, myOutputCols); }
 
-//       public string[] groupByColumns { get { return myGroupByCols; } }
+        public string[] groupByColumns { get { return myGroupByCols; } }
 
-//       public void clear()  {/*stateless*/}
+        public void clear() {/*stateless*/}
 
-//       public IBlueTableSchema getOutputSchema(IBlueTableSchema schema)  {return schema;}
+        public IBlueTableSchema getOutputSchema(IBlueTableSchema schema) { return schema; }
 
-//       #endregion
-//    }
+        #endregion
+    }
 
-//    public class SumGroupByEvaluator<T> : GroupByEvaluator
-//    {
-//       public SumGroupByEvaluator(string outputCol, string[] groupByCols)
-//          : base(outputCol, groupByCols)
-//       { }
+    public class SumGroupByEvaluator<T> : GroupByEvaluator {
+        public SumGroupByEvaluator(string outputCol, string[] groupByCols)
+           : base(outputCol, groupByCols) { }
 
-//       public SumGroupByEvaluator(string colMapEpression, string outputCol, string[] groupByCols)
-//          : base(colMapEpression, outputCol, groupByCols) { }
+        public SumGroupByEvaluator(string colMapEpression, string outputCol, string[] groupByCols)
+           : base(colMapEpression, outputCol, groupByCols) { }
 
-//       public override IBlueRow evaluate(DataPacket dp)
-//       {
-//          myOutputTable = myOutputTable == null ? new GroupByOutputTable(dp.schema) : myOutputTable;
+        public override IBlueRow evaluate(DataPacket dp) {
+            myOutputTable = myOutputTable == null ? new GroupByOutputTable(dp.schema) : myOutputTable;
 
-//          IBlueRow sumRow = BlueDataUtility.computeSum<T>(dp.newRows, myOutputColumn);
-//          return sumRow;
-//       }
-//    }
+            IBlueRow sumRow = BlueDataUtility.computeSum<T>(dp.newRows, myOutputColumn);
+            return sumRow;
+        }
+    }
 
-//    public class MultiColumnSumGroupByEvaluator<T> : IGroupByEvaluator
-//    {
-//       private string[] myOutputCols, myGroupByCols;
-//       public MultiColumnSumGroupByEvaluator(string spaceSeperatedOutputCols, string spaceSeperatedGroupByCols)
-//       {
-//          myOutputCols = spaceSeperatedOutputCols.Split(' ');
-//          myGroupByCols = spaceSeperatedGroupByCols.Split(' ');
-//       }
+    public class MultiColumnSumGroupByEvaluator<T> : IGroupByEvaluator {
+        private string[] myOutputCols, myGroupByCols;
+        public MultiColumnSumGroupByEvaluator(string spaceSeperatedOutputCols, string spaceSeperatedGroupByCols) {
+            myOutputCols = spaceSeperatedOutputCols.Split(' ');
+            myGroupByCols = spaceSeperatedGroupByCols.Split(' ');
+        }
 
-//       #region IGroupByEvaluator Members
+        #region IGroupByEvaluator Members
 
-//       public IBlueRow evaluate(DataPacket packet)  {return BlueDataUtility.computeSum<T>(packet.newRows, myOutputCols);}
+        public IBlueRow evaluate(DataPacket packet) { return BlueDataUtility.computeSum<T>(packet.newRows, myOutputCols); }
 
-//       public string[] groupByColumns { get { return myGroupByCols; } }
+        public string[] groupByColumns { get { return myGroupByCols; } }
 
-//       public void clear()  {/*stateless*/}
+        public void clear() {/*stateless*/}
 
-//       public IBlueTableSchema getOutputSchema(IBlueTableSchema schema)  {return schema;}
+        public IBlueTableSchema getOutputSchema(IBlueTableSchema schema) { return schema; }
 
-//       #endregion
-//    }
+        #endregion
+    }
 
-// }
+}

src/DataStreamSuite/DataStream/Primitives/GroupBy/SumStream.cs

@@ -1,107 +1,89 @@
-// using System;
-// using System.Collections.Generic;
+using System;
+using System.Collections.Generic;
 
-// namespace BCG
-// {
-//    public class NullMapper : IStreamMapper
-//    {
-//       private Dictionary<string, bool> _valid_cols = new Dictionary<string, bool>();
+namespace BCG {
+    public class NullMapper : IStreamMapper {
+        private Dictionary<string, bool> _valid_cols = new Dictionary<string, bool>();
 
-//       public NullMapper(string[] valid_cols)
-//       {
-//          foreach (string c in valid_cols)
-//             _valid_cols[c] = true;
-//          _valid_cols["id"] = true;
-//       }
+        public NullMapper(string[] valid_cols) {
+            foreach (string c in valid_cols)
+                _valid_cols[c] = true;
+            _valid_cols["id"] = true;
+        }
 
-//       public IBlueTableSchema getOutputSchema(IBlueTableSchema input_schema) { return input_schema; }
+        public IBlueTableSchema getOutputSchema(IBlueTableSchema input_schema) { return input_schema; }
 
-//       public IBlueRow map(IBlueRow row)
-//       {
-//          MutableBlueRow out_row = new MutableBlueRow(row);
-//          foreach (string c in row.schema.columnNames)
-//             if (!_valid_cols.ContainsKey(c))
-//                out_row.setValue(c, DBNull.Value);
-//          return out_row;
-//       }
-//    }
+        public IBlueRow map(IBlueRow row) {
+            MutableBlueRow out_row = new MutableBlueRow(row);
+            foreach (string c in row.schema.columnNames)
+                if (!_valid_cols.ContainsKey(c))
+                    out_row.setValue(c, DBNull.Value);
+            return out_row;
+        }
+    }
 
-//    public class SumStream : ViewStream
-//    {
-//       public SumStream(ViewStream stream, string spaceSeparatedOutputCols, string spaceSeparatedGroupByCols)
-//       {
-//          string[] outputCols = spaceSeparatedOutputCols.Split(' ');
-//          string[] groupByCols = spaceSeparatedGroupByCols.Split(' ');
-//          mySetRealSource(new GroupByStream(stream,
-//                                            outputCols.Length == 1 ? new SumGroupByEvaluator(outputCols[0], groupByCols) as IGroupByEvaluator :
-//                                            new MultiColumnSumGroupByEvaluator(spaceSeparatedOutputCols, spaceSeparatedGroupByCols)));
-//       }
+    public class SumStream : ViewStream {
+        public SumStream(ViewStream stream, string spaceSeparatedOutputCols, string spaceSeparatedGroupByCols) {
+            string[] outputCols = spaceSeparatedOutputCols.Split(' ');
+            string[] groupByCols = spaceSeparatedGroupByCols.Split(' ');
+            mySetRealSource(new GroupByStream(stream,
+                                              outputCols.Length == 1 ? new SumGroupByEvaluator(outputCols[0], groupByCols) as IGroupByEvaluator :
+                                              new MultiColumnSumGroupByEvaluator(spaceSeparatedOutputCols, spaceSeparatedGroupByCols)));
+        }
 
-//       public SumStream(ViewStream stream, string[] output_columns)
-//       {
-//          ViewStream rs = new AddColumnsStream(stream, "fake_col=0");
-//          rs = new SumStream(rs, output_columns, new string[] { "fake_col" }, false);
-//          rs = new ProjectStream(rs, "!fake_col");
-//          mySetRealSource(rs);
-//       }
+        public SumStream(ViewStream stream, string[] output_columns) {
+            ViewStream rs = new AddColumnsStream(stream, "fake_col=0");
+            rs = new SumStream(rs, output_columns, new string[] { "fake_col" }, false);
+            rs = new ProjectStream(rs, "!fake_col");
+            mySetRealSource(rs);
+        }
 
-//       public SumStream(ViewStream stream, string space_seperated_output_columns, string space_seperated_groupby_columns, bool force_null_into_other_columns)
-//       {
-//          string[] output_columns = space_seperated_output_columns.Split(' ');
-//          string[] groupby_columns = String.IsNullOrEmpty (space_seperated_groupby_columns) ? new string[]{} : space_seperated_groupby_columns.Split (' ');
-//          mySetRealSource (new SumStream (stream, output_columns, groupby_columns, force_null_into_other_columns));
-//       }
+        public SumStream(ViewStream stream, string space_seperated_output_columns, string space_seperated_groupby_columns, bool force_null_into_other_columns) {
+            string[] output_columns = space_seperated_output_columns.Split(' ');
+            string[] groupby_columns = String.IsNullOrEmpty(space_seperated_groupby_columns) ? new string[] { } : space_seperated_groupby_columns.Split(' ');
+            mySetRealSource(new SumStream(stream, output_columns, groupby_columns, force_null_into_other_columns));
+        }
 
-//       public SumStream(ViewStream stream, string[] output_columns, string[] groupby_columns, bool force_null_into_other_columns)
-//       {
-//          ViewStream rs = null;
-//          groupby_columns = groupby_columns ?? new string[] {};
-//          if (groupby_columns.Length == 0)
-//          {
-//             rs = new AddColumnsStream(stream, "fake_col=0");
-//             rs = new GroupByStream(rs, new MultiColumnSumGroupByEvaluator(output_columns, new string[] {"fake_col"}));
-//             rs = new ProjectStream(rs, "!fake_col");
-//          }
-//          else
-//          {
-//             rs = new GroupByStream(stream, new MultiColumnSumGroupByEvaluator(output_columns, groupby_columns));
-//          }
-//          if (force_null_into_other_columns)
-//          {
-//             List<string> valid_cols = new List<string>(output_columns);
-//             valid_cols.AddRange(groupby_columns);
-//             rs = new MapStream(rs, new NullMapper(valid_cols.ToArray()));
-//          }
-//          mySetRealSource(rs);
-//       }
+        public SumStream(ViewStream stream, string[] output_columns, string[] groupby_columns, bool force_null_into_other_columns) {
+            ViewStream rs = null;
+            groupby_columns = groupby_columns ?? new string[] { };
+            if (groupby_columns.Length == 0) {
+                // rs = new AddColumnsStream(stream, "fake_col=0");
+                // rs = new GroupByStream(rs, new MultiColumnSumGroupByEvaluator(output_columns, new string[] { "fake_col" }));
+                // rs = new ProjectStream(rs, "!fake_col");
+            } else {
+                rs = new GroupByStream(stream, new MultiColumnSumGroupByEvaluator(output_columns, groupby_columns));
+            }
+            if (force_null_into_other_columns) {
+                List<string> valid_cols = new List<string>(output_columns);
+                valid_cols.AddRange(groupby_columns);
+                rs = new MapStream(rs, new NullMapper(valid_cols.ToArray()));
+            }
+            mySetRealSource(rs);
+        }
 
-//    public SumStream (ViewStream stream, string outputCol)
-//    : this (new AddColumnsStream (stream, "fakeCol=0"), outputCol, "fakeCol")
-//    {
-//    }
-// }
+        public SumStream(ViewStream stream, string outputCol)
+        : this(new AddColumnsStream(stream, "fakeCol=0"), outputCol, "fakeCol") {
+        }
+    }
 
-// public class IntSumStream : ViewStream
-// {
-//    public IntSumStream (ViewStream stream, string spaceSeparatedOutputCols, string spaceSeparatedGroupByCols)
-//    {
-//       string[] outputCols = spaceSeparatedOutputCols.Split(' ');
-//       string[] groupByCols = spaceSeparatedGroupByCols.Split(' ');
-//       mySetRealSource(new GroupByStream(stream,
-//                                         outputCols.Length == 1 ? new SumGroupByEvaluator<int>(outputCols[0], groupByCols) as IGroupByEvaluator:
-//                                         new MultiColumnSumGroupByEvaluator<int>(spaceSeparatedOutputCols, spaceSeparatedGroupByCols)));
-//    }
+    public class IntSumStream : ViewStream {
+        public IntSumStream(ViewStream stream, string spaceSeparatedOutputCols, string spaceSeparatedGroupByCols) {
+            string[] outputCols = spaceSeparatedOutputCols.Split(' ');
+            string[] groupByCols = spaceSeparatedGroupByCols.Split(' ');
+            mySetRealSource(new GroupByStream(stream,
+                                              outputCols.Length == 1 ? new SumGroupByEvaluator<int>(outputCols[0], groupByCols) as IGroupByEvaluator :
+                                              new MultiColumnSumGroupByEvaluator<int>(spaceSeparatedOutputCols, spaceSeparatedGroupByCols)));
+        }
 
-// }
+    }
 
 
-//    public class SumStream<T> : ViewStream
-//    {
-//       public SumStream(ViewStream stream, string outputCol, string spaceSeparatedInputCols)
-//       {
-//          string[] inputCols = spaceSeparatedInputCols.Split(' ');
-//          mySetRealSource(new GroupByStream(stream,
-//                                              new SumGroupByEvaluator<T>(outputCol, inputCols)));
-//       }
-//    }
-// }
+    public class SumStream<T> : ViewStream {
+        public SumStream(ViewStream stream, string outputCol, string spaceSeparatedInputCols) {
+            string[] inputCols = spaceSeparatedInputCols.Split(' ');
+            mySetRealSource(new GroupByStream(stream,
+                                                new SumGroupByEvaluator<T>(outputCol, inputCols)));
+        }
+    }
+}

src/DataStreamSuite/DataStream/Primitives/InterRow/RowDiffStream.cs

@@ -0,0 +1,79 @@
+using BCG;
+
+using System.Collections;
+using System.Collections.Generic;
+using System.Linq;
+
+using ZstdSharp.Unsafe;
+
+namespace bts {
+    public class RowDiffStream : ViewStream {
+        private ViewStream _vs;
+        private readonly string _order_col, _col_1, _col_2;
+        private readonly int _diff;
+        private IBlueTableSchema _out_schema;
+        private readonly string _diff_col_name;
+        public RowDiffStream(ViewStream vs, string order_col, string col_1, string col_2, int diff, string diff_col_name) {
+            _vs = vs;
+            _order_col = order_col;
+            _col_1 = col_1;
+            _col_2 = col_2;
+            _diff = diff;
+            _diff_col_name = diff_col_name;
+        }
+
+        private string _diff_col_name_perc {
+            get {
+                return $"{_diff_col_name}_percentage";
+            }
+        }
+
+        protected override void myOnFirstSubscriber() {
+            _vs.onChange += _handle_vs;
+        }
+
+        protected override void myOnLastSubscriber() {
+            _vs.onChange -= _handle_vs;
+        }
+
+        private void _handle_vs(object dp_obj) {
+            var dp = dp_obj as DataPacket;
+            IBlueRowCollection nrows = dp.newRows;
+            _out_schema = BlueDataUtility.add_column(nrows.schema, _diff_col_name, typeof(double));
+            _out_schema = BlueDataUtility.add_column(_out_schema, _diff_col_name_perc, typeof(double));
+            List<IBlueRow> row_list = new List<IBlueRow>();
+            foreach (IBlueRow r in nrows)
+                row_list.Add(r);
+            //OrderBy sorts in ascedning order
+            var sorted_rows = row_list.OrderBy(r => r[_order_col]).ToList();
+            var sorted_row_array = sorted_rows.ToArray();
+            // reverse it to make it descending    
+            sorted_rows.Reverse();
+            //sort rows so we can do the diff
+            OrderedBlueTable rv = new OrderedBlueTable(_out_schema);
+            int i = 0;
+            foreach (IBlueRow r in sorted_rows) {
+                MutableBlueRow nr = new MutableBlueRow(_out_schema, r);
+                //So, in the expression "x - y," where y is subtracted from x, x is the minuend, and y is the subtrahend.
+                // we assume x - y, and name variables like that
+                double x = r.get<double>(_col_1);
+                int y_row_idx = i + _diff;
+                double diff = System.Double.NaN;
+                double perc = System.Double.NaN;
+                if (y_row_idx < sorted_rows.Count()) {
+                    var y = sorted_rows[y_row_idx].get<double>(_col_2);
+                    diff = x - y;
+                    perc = 100.0 * diff / y;
+                }
+                nr.setValue(_diff_col_name, diff);
+                nr.setValue(_diff_col_name_perc, perc);
+                rv.addRow(nr);
+                i++;
+            }
+
+            var dp_out = BlueDataUtility.build_dp_inc(new_rows: rv, myLastValue as DataPacket, true);
+            myFireEvent(dp_out);
+
+        }
+    }
+}

src/DataStreamSuite/DataStream/Primitives/SideEffectStreams/SchemaGuesserStream.cs

@@ -0,0 +1,34 @@
+using BCG;
+
+namespace bts {
+    public class SchemaGuesserStream : ViewStream {
+
+        private IBlueRowCollection _rows = null;
+        private ViewStream _vs;
+        public SchemaGuesserStream(ViewStream vs) {
+            _vs = vs;
+        }
+
+        protected override void myOnFirstSubscriber() {
+            _vs.onChange += _on_data;
+        }
+        protected override void myOnLastSubscriber() {
+            _vs.onChange -= _on_data;
+        }
+
+        private void _on_data(object obj) {
+            DataPacket dp = obj as DataPacket;
+            if (_rows == null) {
+                _rows = BlueDataUtility.get_typed_table(dp.newRows);
+            }
+            var dp_out = BlueDataUtility.build_dp_inc(_rows, myLastValue as DataPacket);
+            myFireEvent(dp_out);
+
+
+        }
+
+
+
+
+    }
+}

src/DataStreamSuite/DataStream/Primitives/Sort/SortRow.cs

@@ -1,115 +1,107 @@
-// using System;
-// using System.Collections;
-// using BCG.Utilities;
-// using BCG.Evaluator.SortExpression;
-
-
-// namespace BCG
-// {
-//    public class SortRow : IComparable, IBlueRow
-//    {
-//       private IBlueRow myRow;
-//       private IEnumerable myColumns;
-//       private IComparer myComparer;
-
-//       public SortRow (IBlueRow row, SortExpression sortExpression)
-//       {
-//          myRow = row;
-//          myColumns = sortExpression.columns;
-//       }
-
-//       public SortRow (IBlueRow row, IEnumerable columns)
-//       {
-//          myRow = row;
-//          myColumns = columns;
-//       }
-
-//       public SortRow (IBlueRow row, IComparer comparer)
-//       {
-//          myRow = row;
-//          myComparer = comparer;
-//       }
-
-//       public IBlueRow internal_row { get { return myRow; } }
-
-//       #region IComparable method
-//       public int CompareTo (object obj)
-//       {  
-//          IBlueRow otherRow = (IBlueRow)obj;
-//          if (myComparer == null)
-//          {
-//             foreach (SortExpression.ColumnCriteria cc in myColumns)
-//             {
-//                int retVal = myIsNull (this[cc.columnName]) ?
-//                   myIsNull (otherRow[cc.columnName]) ? 0 : -1 :
-//                   myIsNull (otherRow[cc.columnName]) ? 1 : ((IComparable)this[cc.columnName]).CompareTo (otherRow[cc.columnName]);
-//                retVal *= cc.ascending ? 1 : -1;
-//                if (retVal != 0)
-//                   return retVal;
-//             }
-//             return 0;
-//          }
-//          else
-//             return myComparer.Compare (this, otherRow);
-//       }
-//       #endregion
-
-//       #region IBlueRow members
-//       public IBlueTableSchema schema{get{return myRow.schema;}}
-//       #endregion
-
-//       #region ICollection members
-//       public int Count {get{return myRow.Count;}}
-
-//       public bool IsSynchronized {get{return false;}}
-
-//       public object SyncRoot {get{return this;}}
-
-//       public void CopyTo (Array array, int index) {myRow.CopyTo (array, index);}
-//       #endregion
-
-//       #region IDictionary members
-//       public bool IsFixedSize {get{return true;}}
-
-//       public bool IsReadOnly {get{return true;}}
-
-//       /// <summary>
-//       /// The key can either be the string corrsponding to the column name or
-//       /// the order of the column
-//       /// </summary>
-//       public object this [object key]
-//       {
-//          get
-//          {
-//             if (key is string)
-//                return this[(string)key];
-//             return this[(int)key];
-//          }
-//          set {throw new NotSupportedException ("The IBlueRow is readonly");}
-//       }
-
-//       public object this [int index]   {get{return myRow[index];}}
-
-//       public object this [string colName] {get{return myRow[colName];}}
-
-//       public ICollection Keys{get{return myRow.Keys;}}
-
-//       public ICollection Values{get{return myRow.Values;}}
-
-//       public void Add (object key, object value)   {throw new NotSupportedException ("The IBlueRow is readonly");}
-
-//       public void Clear () {throw new NotSupportedException ("The IBlueRow is readonly");}
-
-//       public bool Contains (object key){return myRow.Contains (key);}
-
-//       IDictionaryEnumerator IDictionary.GetEnumerator() {return myRow.GetEnumerator();}
-
-//       IEnumerator IEnumerable.GetEnumerator () {return myRow.GetEnumerator();}
-
-//       public void Remove (object key) {throw new NotSupportedException ("The IBlueRow is readonly");}
-//       #endregion
-
-//       private bool myIsNull (object o) {return o == null || o is DBNull;}
-
-//    }
-// }
+using Bts.ColumnSortExpressions;
+
+using System;
+using System.Collections;
+using System.Collections.Generic;
+using System.Linq;
+
+namespace BCG;
+
+public class SortRow : IComparable, IBlueRow {
+   readonly IBlueRow row;
+   readonly IEnumerable<ColumnSortExpression>? columns;
+   readonly IComparer? comparer;
+
+   public SortRow(IBlueRow row, IList<ColumnSortExpression> columns) {
+      this.row = row;
+      this.columns = columns;
+   }
+
+   public SortRow(IBlueRow row, IComparer comparer) {
+      this.row = row;
+      this.comparer = comparer;
+   }
+
+   public IBlueRow internal_row { get { return row; } }
+
+   #region IComparable method
+   public int CompareTo(object? obj) {
+      if (obj is not IBlueRow otherRow) {
+         throw new ArgumentException(
+            $"Expected obj to be of type {typeof(IBlueRow)} (actually {obj?.GetType()})",
+            nameof(obj));
+      }
+      if (comparer is not null) {
+         return comparer.Compare(this, otherRow);
+      }
+      return columns.empty_if_null().Select(Compare).FirstOrDefault(c => c != 0);
+      int Compare(ColumnSortExpression col) {
+         var otherVal = otherRow[col.Name];
+         int rv = myIsNull(this[col.Name]) ?
+            (myIsNull(otherVal) ? 0 : -1) :
+            (myIsNull(otherVal) ? 1 : ((IComparable)this[col.Name]).CompareTo(otherVal));
+         rv *= col.IsAscending ? 1 : -1;
+         return rv;
+      }
+   }
+   #endregion
+
+   #region IBlueRow members
+   public IBlueTableSchema schema { get { return row.schema; } }
+   #endregion
+
+   #region ICollection members
+   public int Count { get { return row.Count; } }
+
+   public bool IsSynchronized { get { return false; } }
+
+   public object SyncRoot { get { return this; } }
+
+   public void CopyTo(Array array, int index) { row.CopyTo(array, index); }
+   #endregion
+
+   #region IDictionary members
+   public bool IsFixedSize { get { return true; } }
+
+   public bool IsReadOnly { get { return true; } }
+
+   /// <summary>
+   /// The key can either be the string corrsponding to the column name or
+   /// the order of the column
+   /// </summary>
+   public object this[object key] {
+      get {
+         if (key is string)
+            return this[(string)key];
+         return this[(int)key];
+      }
+      set { throw new NotSupportedException("The IBlueRow is readonly"); }
+   }
+
+   public object this[int index] => row[index];
+
+   public object this[string colName] => row[colName];
+
+   public ICollection Keys => row.Keys;
+
+   public ICollection Values => row.Values;
+
+   public bool Contains(object key) => row.Contains(key);
+
+   void IDictionary.Add(object key, object? value) =>
+      throw new NotSupportedException("The IBlueRow is readonly");
+
+   void IDictionary.Clear() =>
+      throw new NotSupportedException("The IBlueRow is readonly");
+
+   IDictionaryEnumerator IDictionary.GetEnumerator() { return row.GetEnumerator(); }
+
+   IEnumerator IEnumerable.GetEnumerator() { return row.GetEnumerator(); }
+
+   public void Remove(object key) { throw new NotSupportedException("The IBlueRow is readonly"); }
+   #endregion
+
+   bool myIsNull(object? o) =>
+      o is null || o == null || o is DBNull;
+
+}

src/DataStreamSuite/DataStream/Primitives/Sort/SortStream.cs

@@ -1,129 +1,99 @@
-// using System;
-// using System.Collections;
-// using BCG.Evaluator.SortExpression;
+using Bts.ColumnSortExpressions;
 
+using System;
+using System.Collections;
+using System.Collections.Generic;
+using System.Diagnostics.CodeAnalysis;
 
-// namespace BCG
-// {
-// 	/// <summary>
-// 	/// Defines a Sort stream, with performance of O(n) for all the increments. It uses simple merge sort
-// 	/// to merge the additions to the accumulates table
-// 	/// </summary>
-// 	public class SortStream : ViewStream
-// 	{
-//       private ViewStream myStream;
-//       private IEnumerable mySortCols;
-//       private IComparer myComparer;
-//       private DataPacket myLastFiredPacket;
-//       private DataPacket myLastRecievedPacket;
+namespace BCG;
 
-//       public SortStream (ViewStream stream, string sortExpression)
-//       {
-//          myStream = stream;
-//          mySetSortExpression (sortExpression);
-//       }
+/// <summary>
+/// Defines a Sort stream, with performance of O(n) for all the increments. It uses simple merge sort
+/// to merge the additions to the accumulates table
+/// </summary>
+public class SortStream : ViewStream {
+   ViewStream stream;
+   IList<ColumnSortExpression> sortCols;
+   IComparer? comparer;
+   DataPacket? lastFiredPacket;
+   DataPacket? lastReceivedPacket;
 
-//       public SortStream (ViewStream vs, string colName, bool isAsc)
-//       {
-//          myStream = vs;
-//          string sortExpression = isAsc ? "+" + colName : "-" + colName;
-//          mySetSortExpression (sortExpression);
-//       }
+   public SortStream(ViewStream stream, string sortExpression) {
+      this.stream = stream;
+      SortExpression = sortExpression;
+   }
 
-//       public SortStream (ViewStream vs, IComparer comparer)
-//       {
-//          myStream = vs;
-//          myComparer = comparer;
-//       }
+   public SortStream(ViewStream vs, string colName, bool isAsc) {
+      stream = vs;
+      SortExpression = isAsc ? "+" + colName : "-" + colName;
+   }
 
-//       public string sortExpression
-//       {
-//          set
-//          {
-//             myComparer = null;
-//             myLastFiredPacket = null;
-//             mySetSortExpression (value);   
-//             myChangeHandler (myLastRecievedPacket);
-//          }
-//       }
+   public SortStream(ViewStream stream, IComparer comparer) {
+      this.stream = stream;
+      this.comparer = comparer;
+      sortCols = Array.Empty<ColumnSortExpression>();
+   }
 
-//       public IComparer comparer
-//       {
-//          set
-//          {
-//             myComparer = value;
-//             myLastFiredPacket = null;
-//             myChangeHandler(myLastRecievedPacket);
-//          }
-//       }
+   [MemberNotNull(nameof(sortCols))]
+   public string SortExpression {
+      set {
+         comparer = null;
+         lastFiredPacket = null;
+         sortCols = ColumnSortExpressionInterpreter.Evaluate(value);
+         if (lastReceivedPacket is not null) {
+            myChangeHandler(lastReceivedPacket);
+         }
+      }
+   }
 
-//       private void mySetSortExpression (string sortExpression)
-//       {
-//          syntax parser = 
-//             new syntax();
-         
-//          mySortCols = ((SortExpression)
-//             parser.Parse (sortExpression)).columns;
-//       }
+   public IComparer Comparer {
+      set {
+         comparer = value;
+         lastFiredPacket = null;
+         if (lastReceivedPacket is not null) {
+            myChangeHandler(lastReceivedPacket);
+         }
+      }
+   }
 
-//       protected override void myOnFirstSubscriber()
-//       {
-//          myStream.onChange += new DataStream.ChangeDelegate (myChangeHandler); 
-//       }
+   public void myChangeHandlerTest(object o) =>
+      myChangeHandler(o);
 
-//       protected override void myOnLastSubscriber()
-//       {
-//          myInit ();
-//          myStream.onChange -= new DataStream.ChangeDelegate (myChangeHandler); 
-//       }
+   [MemberNotNull(nameof(lastFiredPacket))]
+   internal void myChangeHandler(object o) {
+      lastReceivedPacket = (DataPacket)o;
+      // First callback, so we need to sort the whole table, this is the
+      // only time we will do that for all the other sorting operations would
+      // be sorting the increments and inserting them into the whole list
+      if (lastFiredPacket is null) {
+         lastFiredPacket = new() {
+            newRows = comparer is null ?
+               new SortedTable(lastReceivedPacket.newRows, sortCols) :
+               new SortedTable(lastReceivedPacket.newRows, comparer)
+         };
+         lastFiredPacket.addedRows = new OrderedBlueTable(lastFiredPacket.newRows);
+         lastFiredPacket.deletedRows = new BlueTable(lastFiredPacket.newRows.schema);
+         myFireEvent(lastFiredPacket.Clone());
+         return;
+      }
+      // Deletes
+      lastFiredPacket.deletedRows =
+         comparer == null ?
+         new SortedTable(lastReceivedPacket.deletedRows, sortCols) :
+         new SortedTable(lastReceivedPacket.deletedRows, comparer);
+      ((SortedTable)lastFiredPacket.newRows).deleteRows(lastFiredPacket.deletedRows);
+      // Adds
+      lastFiredPacket.addedRows = new BlueTable(lastReceivedPacket.addedRows);
+      ((SortedTable)lastFiredPacket.newRows).addRows(lastFiredPacket.addedRows);
+      myFireEvent(lastFiredPacket.Clone());
+   }
 
-//       internal void myChangeHandler (object o)
-//       {
-//          myLastRecievedPacket = (DataPacket)o;
-//          //first callback, so we need to sort the whole table, this is the
-//          //only time we will do that for all the other sorting operations would
-//          //be sorting the increments and inserting them into the whole list
-//          if (myLastFiredPacket == null)
-//          {
-//             myLastFiredPacket = new DataPacket ();
-//             myLastFiredPacket.newRows = 
-//                myComparer == null ?
-//                new SortedTable (myLastRecievedPacket.newRows, mySortCols) :
-//                new SortedTable (myLastRecievedPacket.newRows, myComparer);
-//             myLastFiredPacket.addedRows = new OrderedBlueTable(myLastFiredPacket.newRows);
-//             myLastFiredPacket.deletedRows = new BlueTable (myLastFiredPacket.newRows.schema);
-//             myFireEvent (myLastFiredPacket.Clone());
-//             return;
-//          }
-//          myHandleDeletes (myLastRecievedPacket);
-//          myHandleAdds (myLastRecievedPacket);
-//          myFireEvent (myLastFiredPacket.Clone());
-//       }
+   protected override void myOnFirstSubscriber() =>
+      stream.onChange += myChangeHandler;
 
-//       public void myChangeHandlerTest(object o)
-//       {
-//           myChangeHandler(o);
-//       }
-
-//       private void myHandleDeletes (DataPacket dp)
-//       {
-//          myLastFiredPacket.deletedRows = 
-//             myComparer == null ?
-//             new SortedTable (dp.deletedRows, mySortCols) :
-//             new SortedTable (dp.deletedRows, myComparer);
-//          ((SortedTable)(myLastFiredPacket.newRows)).deleteRows (myLastFiredPacket.deletedRows);
-//       }
-
-//       private void myHandleAdds (DataPacket dp)
-//       {
-//          myLastFiredPacket.addedRows = new BlueTable (dp.addedRows);
-//          ((SortedTable)(myLastFiredPacket.newRows)).addRows (myLastFiredPacket.addedRows);
-//       }
-
-//       private void myInit ()
-//       {
-//          myLastFiredPacket = null;
-//          myLastRecievedPacket = null;
-//       }
-// 	}
-// }
+   protected override void myOnLastSubscriber() {
+      lastFiredPacket = null;
+      lastReceivedPacket = null;
+      stream.onChange -= myChangeHandler;
+   }
+}

src/DataStreamSuite/DataStream/Primitives/Sort/SortedTable.cs

@@ -1,136 +1,108 @@
-// using System;
-// using System.Collections;
-// using BCG.Evaluator.SortExpression;
-
-// namespace BCG
-// {
-//    /// <summary>
-//    /// Represents an IBlueRowCollection with the rows sorted according to the
-//    /// 
-//    /// </summary>
-//    public class SortedTable : IBlueRowCollection 
-//    {
-//       private ArrayList myRows = new ArrayList ();
-//       private Hashtable myIndex = new Hashtable ();
-//       private IBlueTableSchema mySchema;
-//       private IEnumerable mySortCols;
-//       private IComparer myComparer;
-
-//       #region SortedTable method(s)
-//       public void addRows (IBlueRowCollection rows)
-//       {
-//          //requires merger of the addedRows to the already existing ones
-//          if (rows.Count == 0)
-//             //at times there are calls with 0 additions, in that case
-//             //we ofcourse need no processing
-//             return;
-//          SortedTable addedRows = 
-//             myComparer == null ? new SortedTable (rows, mySortCols) :
-//                new SortedTable (rows, myComparer);
-//          int addedRowsIndex = 0;
-//          SortRow newRow = addedRows.getRowByIndex (addedRowsIndex);
-//          for (int i = 0; i < myRows.Count; i ++)
-//             if (newRow.CompareTo(myRows[i]) <= 0)
-//             {
-//                myRows.Insert (i, newRow);
-//                myIndex[newRow["id"]] = newRow;
-//                if (++addedRowsIndex == addedRows.Count)
-//                   return;
-//                newRow = addedRows.getRowByIndex (addedRowsIndex);
-//             }
-//          if (addedRowsIndex < addedRows.Count)
-//             for (int i = addedRowsIndex; i < addedRows.Count; i++)
-//             {
-//                newRow = addedRows.getRowByIndex (i);
-//                myRows.Add (newRow);
-//                myIndex[newRow["id"]] = newRow;
-//             }
-//       }
-
-//       public void deleteRows (IBlueRowCollection deletedRows)
-//       {
-//          foreach (IBlueRow deletedRow in deletedRows)
-//          {
-//             IBlueRow row = (SortRow)myIndex[deletedRow["id"]];
-//             myRows.Remove (row);
-//             myIndex.Remove (row["id"]);
-//          }
-//       }
-
-//       public SortRow getRowByIndex (int index)  {return (SortRow)myRows[index];}
-//       #endregion
-
-//       #region Constructor(s)
-//       /// <summary>
-//       /// Sorts the rows using sortExpression as the sorting criteria
-//       /// </summary>
-//       /// <param name="rows">
-//       /// Rows to be inserted into the table
-//       /// </param>
-//       /// <param name="sortExpression">
-//       /// Sorting criteria
-//       /// </param>
-//       public SortedTable (IBlueRowCollection rows, SortExpression sortExpression)
-//       {
-//          mySortCols = sortExpression.columns;
-//          myConstructorHelper (rows);
-//       }
-
-//       public SortedTable (IBlueRowCollection rows, IEnumerable sortCols)
-//       {
-//          mySortCols = sortCols;
-//          myConstructorHelper (rows);
-//       }
-
-//       public SortedTable (IBlueRowCollection rows, IComparer comparer)
-//       {
-//          myComparer = comparer;
-//          myConstructorHelper (rows);
-//       }
-
-//       private void myConstructorHelper (IBlueRowCollection rows)
-//       {
-//          mySchema = rows.schema;
-//          foreach (IBlueRow row in rows)
-//          {
-//             SortRow newRow = 
-//                myComparer == null ?
-//                new SortRow (row, mySortCols) : new SortRow (row, myComparer);
-//             myRows.Add (newRow);
-//             myIndex[row["id"]] = newRow;
-//          }
-//          myRows.Sort ();
-//       }
-
-//       #endregion
-
-//       #region IBlueRowCollections members
-//       public IBlueRow this[int id] 
-//       {
-//          get
-//          {
-//             if (myIndex[id] == null)
-//                throw new BlueDataException ("ID " + id.ToString() + " does not exist in table " + mySchema.tableName);
-//             return (IBlueRow)myIndex[id];
-//          }
-//       }
-//       public IBlueTableSchema schema {get{return mySchema;}}
-
-//       public bool containsID (int id)  {return myIndex[id] != null;}
-
-//       #endregion
-
-//       #region ICollection members
-//       public IEnumerator GetEnumerator () {return myRows.GetEnumerator ();}
-
-//       public bool IsSynchronized {get{return false;}}
-
-//       public object SyncRoot {get{return this;}}
-
-//       public void CopyTo (Array array, int index){myRows.CopyTo (array, index);}
-
-//       public int Count{get{return myRows.Count;}}
-//          #endregion
-//    }
-
-// }
+using System;
+using System.Collections;
+using System.Collections.Generic;
+using System.Diagnostics.CodeAnalysis;
+
+using Bts.ColumnSortExpressions;
+
+namespace BCG;
+
+/// <summary>
+/// Represents an IBlueRowCollection with the rows sorted according to the
+/// </summary>
+public class SortedTable : IBlueRowCollection {
+   readonly List<SortRow> rows = new(0);
+   readonly Dictionary<int, IBlueRow> rowIndex = new(0);
+   readonly IList<ColumnSortExpression> sortCols = Array.Empty<ColumnSortExpression>();
+   readonly IComparer? comparer;
+
+   /// <summary>
+   /// Sorts the rows using sortExpression as the sorting criteria
+   /// </summary>
+   /// <param name="rows">Rows to be inserted into the table</param>
+   /// <param name="sortCols">Sorting criteria</param>
+   public SortedTable(IBlueRowCollection rows, IList<ColumnSortExpression> sortCols) {
+      this.sortCols = sortCols;
+      init(rows);
+   }
+
+   public SortedTable(IBlueRowCollection rows, IComparer comparer) {
+      this.comparer = comparer;
+      init(rows);
+   }
+
+   public int Count => rows.Count;
+   public bool IsSynchronized => false;
+   public IBlueTableSchema schema { get; private set; }
+   public object SyncRoot => this;
+   public IBlueRow this[int id] => rowIndex[id]
+      ?? throw new BlueDataException(
+         $"ID '{id}' does not exist in table '{schema?.tableName}'");
+
+   public void addRows(IBlueRowCollection rows) {
+      // Requires merger of the addedRows to the already existing ones
+      if (rows.Count == 0) {
+         // At times there are calls with 0 additions, in that case
+         // we of course need no processing
+         return;
+      }
+      SortedTable addedRows =
+         comparer == null ? new SortedTable(rows, sortCols) :
+            new SortedTable(rows, comparer);
+      int addedRowsIndex = 0;
+      SortRow newRow = addedRows.getRowByIndex(addedRowsIndex);
+      for (int i = 0; i < this.rows.Count; i++) {
+         if (newRow.CompareTo(this.rows[i]) <= 0) {
+            this.rows.Insert(i, newRow);
+            rowIndex[(int)newRow["id"]] = newRow;
+            if (++addedRowsIndex == addedRows.Count) {
+               return;
+            }
+            newRow = addedRows.getRowByIndex(addedRowsIndex);
+         }
+      }
+      if (addedRowsIndex < addedRows.Count) {
+         for (int i = addedRowsIndex; i < addedRows.Count; i++) {
+            newRow = addedRows.getRowByIndex(i);
+            this.rows.Add(newRow);
+            rowIndex[(int)newRow["id"]] = newRow;
+         }
+      }
+   }
+
+   public void deleteRows(IBlueRowCollection deletedRows) {
+      foreach (IBlueRow deletedRow in deletedRows) {
+         var row = rowIndex[(int)deletedRow["id"]!] as SortRow
+            ?? throw new InvalidOperationException("Index did not return IBlueRow");
+         rows.Remove(row);
+         rowIndex.Remove((int)row["id"]!);
+      }
+   }
+
+   public SortRow getRowByIndex(int index) =>
+      rows[index]!;
+
+   public bool containsID(int id) =>
+      rowIndex[id] is not null;
+
+   public IEnumerator GetEnumerator() =>
+      rows.GetEnumerator();
+
+   public void CopyTo(Array array, int index) {
+      for (int i = 0; i < Count; i++) {
+         array.SetValue(rows[i], index + i);
+      }
+   }
+
+   [MemberNotNull(nameof(schema))]
+   void init(IBlueRowCollection rows) {
+      schema = rows.schema;
+      foreach (IBlueRow row in rows) {
+         SortRow newRow = comparer is null ?
+            new(row, sortCols) : new(row, comparer);
+         this.rows.Add(newRow);
+         rowIndex[(int)row["id"]!] = newRow;
+      }
+      this.rows.Sort();
+   }
+}

src/DataStreamSuite/DataStream/Primitives/Top/TopStream.cs

@@ -1,49 +1,43 @@
-// using System;
+using System;
 
-// namespace BCG
-// {
-//    public class TopStream : MethodDrivenStream
-//    {
-//       private int myTopCount;
+namespace BCG {
+    public class TopStream : MethodDrivenStream {
+        private int myTopCount;
 
-//       public TopStream (ViewStream stream, int topCount) : base (stream)  {myTopCount = topCount;}
+        public TopStream(ViewStream stream, int topCount) : base(stream) { myTopCount = topCount; }
 
-//       public TopStream (ViewStream stream, 
-//          int topCount, 
-//          string sortExpression) 
-//          :
-//          this (sortExpression == null? 
-//          stream :
-//          new SortStream (stream, sortExpression), topCount)
-//       {}
+        public TopStream(ViewStream stream,
+           int topCount,
+           string sortExpression)
+           :
+           this(sortExpression == null ?
+           stream :
+           new SortStream(stream, sortExpression), topCount) { }
 
-//       protected override void myOnTableClear () 
-//       {
-//          myLastFiredPacket.deletedRows = myLastFiredPacket.newRows;
-//          myLastFiredPacket.newRows = new OrderedBlueTable (myLastFiredPacket.deletedRows.schema);
-//          myLastFiredPacket.addedRows = new OrderedBlueTable (myLastFiredPacket.deletedRows.schema);
-//       }
+        protected override void myOnTableClear() {
+            myLastFiredPacket.deletedRows = myLastFiredPacket.newRows;
+            myLastFiredPacket.newRows = new OrderedBlueTable(myLastFiredPacket.deletedRows.schema);
+            myLastFiredPacket.addedRows = new OrderedBlueTable(myLastFiredPacket.deletedRows.schema);
+        }
 
-//       protected override void myHandleFirstCall () 
-//       {
-//          myLastFiredPacket = new DataPacket ();
-//          myLastFiredPacket.newRows = new OrderedBlueTable (myLastArrivedPacket.newRows, myTopCount);
-//          myLastFiredPacket.addedRows = new OrderedBlueTable(myLastFiredPacket.newRows);
-//          myLastFiredPacket.deletedRows = new OrderedBlueTable (myLastFiredPacket.newRows.schema);
-//       }
+        protected override void myHandleFirstCall() {
+            myLastFiredPacket = new DataPacket();
+            myLastFiredPacket.newRows = new OrderedBlueTable(myLastArrivedPacket.newRows, myTopCount);
+            myLastFiredPacket.addedRows = new OrderedBlueTable(myLastFiredPacket.newRows);
+            myLastFiredPacket.deletedRows = new OrderedBlueTable(myLastFiredPacket.newRows.schema);
+        }
 
-//       protected override void myHandleIncrementalAdd ()     {}
+        protected override void myHandleIncrementalAdd() { }
 
-//       protected override void myHandleIncrementalDelete()   
-//       {
-//          IBlueRowCollection oldRows = myLastFiredPacket.newRows;
-//          myLastFiredPacket.newRows = new OrderedBlueTable (myLastArrivedPacket.newRows, myTopCount);
-//          BlueDataUtility.computeDifferences (myLastFiredPacket.newRows,
-//             oldRows,
-//             myLastFiredPacket.schema.columnNames,
-//             out myLastFiredPacket.deletedRows,
-//             out myLastFiredPacket.addedRows);
-//          myFireSwitch = myLastFiredPacket.addedRows.Count + myLastFiredPacket.deletedRows.Count == 0 ? false : true;
-//       }
-//    }
-// }
+        protected override void myHandleIncrementalDelete() {
+            IBlueRowCollection oldRows = myLastFiredPacket.newRows;
+            myLastFiredPacket.newRows = new OrderedBlueTable(myLastArrivedPacket.newRows, myTopCount);
+            BlueDataUtility.computeDifferences(myLastFiredPacket.newRows,
+               oldRows,
+               myLastFiredPacket.schema.columnNames,
+               out myLastFiredPacket.deletedRows,
+               out myLastFiredPacket.addedRows);
+            myFireSwitch = myLastFiredPacket.addedRows.Count + myLastFiredPacket.deletedRows.Count == 0 ? false : true;
+        }
+    }
+}

src/DataStreamSuite/DataStreamCore/BlueData/BlueDataUtility.cs

@@ -6,6 +6,7 @@ using System.Reflection;
 using System.IO;
 using System.Diagnostics;
 using System.Linq;
+using System.Globalization;
 
 //using BCG.Utilities;
 using BCG.Utilities.General;
@@ -48,6 +49,12 @@ namespace BCG {
             rv.Add(r.get<T>(col));
          return rv;
       }
+      public static IEnumerable<T> get_colvalues<T>(IBlueRowCollection rows, string col) {
+         List<T> rv = new List<T>();
+         foreach (IBlueRow r in rows)
+            rv.Add(r.get<T>(col));
+         return rv;
+      }
 
       public static IDictionary<T, IEnumerable<IBlueRow>> group_by<T>(IBlueRowCollection rows, string gp_col) {
          Dictionary<T, List<IBlueRow>> rv = new Dictionary<T, List<IBlueRow>>();
@@ -241,6 +248,165 @@ namespace BCG {
          return new BlueTableSchema(table_name, cis.ToArray());
       }
 
+      public static TypeCode convert_column_type(IEnumerable<string> col_vals) {
+         // bool
+         bool success = true;
+         foreach (var val in col_vals) {
+            bool parsed_val;
+            success = success && Boolean.TryParse(val, out parsed_val);
+            if (!success)
+               break;
+         }
+         if (success)
+            return TypeCode.Boolean;
+         // int
+         success = true;
+         foreach (var val in col_vals) {
+            int parsed_val;
+            success = success && Int32.TryParse(val, out parsed_val);
+            if (!success)
+               break;
+         }
+         if (success)
+            return TypeCode.Int32;
+         // try double
+         success = true;
+         foreach (var val in col_vals) {
+            double parsed_val;
+            success = success && Double.TryParse(val, out parsed_val);
+            if (!success)
+               break;
+         }
+         if (success)
+            return TypeCode.Double;
+         // try DateTime
+         success = true;
+         foreach (var val in col_vals) {
+            DateTime parsed_val;
+            success = success && DateTime.TryParse(val, out parsed_val);
+            if (!success)
+               break;
+         }
+         if (success)
+            return TypeCode.DateTime;
+         bts.Log.assert(false);
+         return TypeCode.Empty;
+      }
+
+      // public static Type guess_type(string val) {
+      //    //DateTime
+      //    {
+      //       DateTime dt;
+      //       var success = DateTime.TryParseExact(val, "yyyy-MM-dd HH:mm:ss", null, DateTimeStyles.AllowWhiteSpaces, out dt);
+      //       if (success) {
+      //          return typeof(DateTime);
+      //       }
+      //       DateTimeOffset dtoff;
+      //       success = DateTimeOffset.TryParseExact(val, "yyyy-MM-dd HH:mm:sszzz",
+      //                                              null, System.Globalization.DateTimeStyles.None, out dtoff);
+
+      //       if (success) {
+      //          return typeof(DateTime);
+      //       }
+      //    }
+      //    //int
+      //    {
+      //       int i;
+      //       var success = Int32.TryParse(val, out i);
+      //       if (success) {
+      //          return typeof(Int32);
+      //       }
+      //    }
+      //    //double
+      //    {
+      //       double d;
+      //       var success = Double.TryParse(val, out d);
+      //       if (success) {
+      //          return typeof(Double);
+      //       }
+      //    }
+      //    //bool
+      //    {
+      //       bool b;
+      //       var success = Boolean.TryParse(val, out b);
+      //       if (success) {
+      //          return typeof(bool);
+      //       }
+      //    }
+      //    //string
+      //    return typeof(string);
+      // }
+
+      private static Type[] _parsable_types = { typeof(DateTime), typeof(int), typeof(double), typeof(bool) };
+      public static Type guess_type(string val) {
+         foreach (var t in _parsable_types) {
+            var type_val = convert_to_type(t, val);
+            if (type_val != null)
+               return t;
+         }
+         return typeof(string);
+
+      }
+
+      public static IBlueTableSchema get_typed_schema(IBlueRowCollection rows) {
+         // just use one row to guess
+         List<ColumnInfo> cis = new List<ColumnInfo>();
+         foreach (IBlueRow r in rows) {
+            foreach (var ci in r.schema.columnInfo) {
+               if (ci.colType != typeof(System.String)) {
+                  cis.Add(ci);
+                  continue;
+               }
+               Type type = guess_type(r[ci.colName] as string);
+               cis.Add(new ColumnInfo(ci.colName, ci.colNo, type));
+            }
+            return new BlueTableSchema(cis.ToArray(), rows.schema.tableName);
+         }
+         return null;
+      }
+
+      static object convert_to_type(Type t, string value) {
+         // Attempt to convert the string to the specified type
+         try {
+            return Convert.ChangeType(value, t);
+         } catch (Exception) {
+            if (t == typeof(double)) {
+               // some doubles come with currency and ',' we clean that up here (there might be better ways to do this)
+               string cleaned_val = new string(value.Where(c => c != '$' && c != ',').ToArray());
+               cleaned_val = cleaned_val.Replace('−', '-');
+               double rv_d;
+               if (Double.TryParse(cleaned_val, out rv_d))
+                  return rv_d;
+               else
+                  return null;
+            }
+            return null;
+            // Handle conversion failure gracefully
+            // throw new ArgumentException("Failed to convert the string to the specified type.");
+         }
+      }
+
+      // converts a row where all columns are string types to a schema where the best types are guesed
+      public static IBlueRowCollection get_typed_table(IBlueRowCollection string_rows) {
+         Func<IBlueRow, IBlueTableSchema, IBlueRow> get_typed_row = (IBlueRow string_row, IBlueTableSchema typed_schema) => {
+            MutableBlueRow rv = new MutableBlueRow(typed_schema);
+            foreach (ColumnInfo ci in typed_schema.columnInfo) {
+               var typed_val = convert_to_type(ci.colType, value: string_row[ci.colName].ToString());
+               rv.setValue(ci.colName, typed_val);
+            }
+            return rv;
+         };
+         IBlueTableSchema typed_schema = get_typed_schema(string_rows);
+         var rv = new BlueTable(typed_schema);
+         foreach (IBlueRow r in string_rows) {
+            var typed_row = get_typed_row(r, typed_schema);
+            rv.addRow(typed_row);
+         }
+         return rv;
+      }
+
+
+
       /// <summary>
       /// Gets the Hashcode of the row with the boolean indicating whether the order of the rows should
       /// be alphabetic or not
@@ -2044,5 +2210,33 @@ namespace BCG {
       }
 
       #endregion
+
+      public class RowComparer<T> : IComparer<IBlueRow> {
+         private readonly IEnumerable<Tuple<string, bool>> _sort_cols;
+         public RowComparer(IEnumerable<Tuple<string, bool>> sort_cols) {
+            _sort_cols = sort_cols;
+         }
+         public int Compare(IBlueRow? x, IBlueRow? y) {
+            Func<object, object, int> compare_cols = (o1, o2) => {
+               return (o1 as IComparable).CompareTo(o2);
+            };
+            foreach (var c in _sort_cols) {
+               int i = compare_cols(x[c.Item1], y[c.Item1]);
+
+            }
+            return 1;
+         }
+
+      }
+      public static IList<IBlueRow> get_sorted_row(IBlueRowCollection rows, IEnumerable<Tuple<string, bool>> sort_cols) {
+         List<IBlueRow> row_list = new List<IBlueRow>();
+         foreach (IBlueRow r in rows) {
+            row_list.Add(r);
+         }
+         var row_comparer = new RowComparer<IBlueRow>(sort_cols);
+         ParallelSort.QuicksortSequential<IBlueRow>(row_list, row_comparer);
+         return row_list;
+
+      }
    }
 }

src/DataStreamSuite/DataStreamExpressions/DataStreamExpressions.csproj

@@ -8,7 +8,7 @@
 
   <ItemGroup>
     <PackageReference Include="Google.Protobuf" Version="3.24.3" />
-    <PackageReference Include="YaccLexTools" Version="1.2.0" />
+    <PackageReference Include="YaccLexTools" Version="1.2.2" />
   </ItemGroup>
 
   <ItemGroup>

src/DataStreamSuite/DataStreamPublisher/Program.cs

@@ -6,7 +6,10 @@ using Bts.DataStreamExpressions;
 using System.Collections.Generic;
 
 using BCG;
-using System.Threading;
+using System.IO;
+using BCG.Utilities.General;
+using bts;
+using System.Linq;
 
 
 bts.Log.Gen.info($"Starting DSPublisher");
@@ -57,6 +60,8 @@ static void start_echo_client() {
 void _publish_streams() {
    PubTableBroker pb_broker = new PubTableBroker();
    CloudStreamManager csm = new CloudStreamManager();
+   string csv_folder = @"C:\Temp\csvs_topublish";
+   CSVPublisher csv_publisher = new CSVPublisher(csv_folder, pb_broker);
    // get it from config
    Dictionary<string, string> streams_to_publish = new Dictionary<string, string>() {
         //{"bbo","bbo()"},
@@ -65,6 +70,7 @@ void _publish_streams() {
         //{"books","books()"},
        {"seconds_1_10","second(1,10)"},
         {"literal_420","literal(\"x\", \"420.0\")"},
+        {"tradebar_30","tradebars(30)"},
         {"mongodb_testcollection", "mongodb(\"mongodb://192.168.1.80:27017\",\"local\",\"test_collection\")"}
         };
    //  {"bbo_ts","timeSeries(bbo())"}
@@ -88,4 +94,37 @@ void _publish_streams() {
 
 void on_change(object obj) {
    bts.Log.Gen.info($"DSPublisher on_change dp: {obj}");
+}
+
+public class CSVPublisher : IWorker {
+   private IDictionary<string, ViewPublisher> _pubs = new Dictionary<string, ViewPublisher>();
+   // read from config
+   private readonly string _csv_directory = @"C:\Temp\csvs_topublish\";
+
+   private PubTableBroker _pub_table_broker;
+   public CSVPublisher(string csv_directory, PubTableBroker pub_table_broker) {
+      _csv_directory = csv_directory;
+      _pub_table_broker = pub_table_broker;
+      MasterWorker.getInstance(false).addWorker(this, TimeSpan.FromSeconds(1));
+   }
+   public void doWork() {
+      var file_names = Directory.GetFiles(_csv_directory);
+      foreach (var fn in file_names) {
+         if (!_pubs.ContainsKey(fn)) {
+            _handle_new_file(fn);
+         }
+      }
+   }
+
+   private void _handle_new_file(string file_name) {
+      if (file_name.EndsWith(".csv")) {
+         ViewStream vs = new CSVReaderStream(file_name);
+         vs = new SchemaGuesserStream(vs);
+         var toks = file_name.Split('\\');
+         string table_name = toks.Last();
+         table_name = table_name.Remove(table_name.Length - ".csv".Length);
+         ViewPublisher vp = new ViewPublisher(table_name, _pub_table_broker, vs);
+         _pubs.Add(file_name, vp);
+      }
+   }
 }

src/DataStreamSuite/DataStreamSuite.sln

@@ -1,4 +1,4 @@
-Microsoft Visual Studio Solution File, Format Version 12.00
+Microsoft Visual Studio Solution File, Format Version 12.00
 # Visual Studio Version 17
 VisualStudioVersion = 17.0.31903.59
 MinimumVisualStudioVersion = 10.0.40219.1
@@ -38,6 +38,10 @@ Project("{FAE04EC0-301F-11D3-BF4B-00C04F79EFBC}") = "DataStreamTests", "DataStre
 EndProject
 Project("{FAE04EC0-301F-11D3-BF4B-00C04F79EFBC}") = "SharedStreams", "SharedStreams\SharedStreams.csproj", "{C1EBC423-4668-43D4-B7E5-A291086BA823}"
 EndProject
+Project("{FAE04EC0-301F-11D3-BF4B-00C04F79EFBC}") = "Bts.ColumnExpressions", "Bts.ColumnExpressions\Bts.ColumnExpressions.csproj", "{134C3915-CB2B-494C-BD62-E0B18250CBF5}"
+EndProject
+Project("{FAE04EC0-301F-11D3-BF4B-00C04F79EFBC}") = "Bts.ColumnSortExpressions", "Bts.ColumnSortExpressions\Bts.ColumnSortExpressions.csproj", "{B038CB72-4E81-4AF1-976F-ED8655C2C3D6}"
+EndProject
 Global
 	GlobalSection(SolutionConfigurationPlatforms) = preSolution
 		Debug|Any CPU = Debug|Any CPU
@@ -119,5 +123,13 @@ Global
 		{C1EBC423-4668-43D4-B7E5-A291086BA823}.Debug|Any CPU.Build.0 = Debug|Any CPU
 		{C1EBC423-4668-43D4-B7E5-A291086BA823}.Release|Any CPU.ActiveCfg = Release|Any CPU
 		{C1EBC423-4668-43D4-B7E5-A291086BA823}.Release|Any CPU.Build.0 = Release|Any CPU
+		{134C3915-CB2B-494C-BD62-E0B18250CBF5}.Debug|Any CPU.ActiveCfg = Debug|Any CPU
+		{134C3915-CB2B-494C-BD62-E0B18250CBF5}.Debug|Any CPU.Build.0 = Debug|Any CPU
+		{134C3915-CB2B-494C-BD62-E0B18250CBF5}.Release|Any CPU.ActiveCfg = Release|Any CPU
+		{134C3915-CB2B-494C-BD62-E0B18250CBF5}.Release|Any CPU.Build.0 = Release|Any CPU
+		{B038CB72-4E81-4AF1-976F-ED8655C2C3D6}.Debug|Any CPU.ActiveCfg = Debug|Any CPU
+		{B038CB72-4E81-4AF1-976F-ED8655C2C3D6}.Debug|Any CPU.Build.0 = Debug|Any CPU
+		{B038CB72-4E81-4AF1-976F-ED8655C2C3D6}.Release|Any CPU.ActiveCfg = Release|Any CPU
+		{B038CB72-4E81-4AF1-976F-ED8655C2C3D6}.Release|Any CPU.Build.0 = Release|Any CPU
 	EndGlobalSection
 EndGlobal

src/DataStreamSuite/PubSubServer/Program.cs

@@ -21,7 +21,9 @@ namespace BCG {
          if (run_echo_tcp_server_only) {
             Console.WriteLine("Running as TCP Echo server, Hit enter to continue");
             Console.ReadLine();
-            SimpleTCPServer sts = new SimpleTCPServer(12345);
+            var port = 12345;
+            Console.WriteLine($"ready to listen to port: {port}");
+            SimpleTCPServer sts = new SimpleTCPServer(port);
             sts.init();
             Task.Delay(1000000000).Wait();
          }

src/DataStreamSuite/SharedStreams/Business/MarketData/Charting/TradeBarsStream.cs

@@ -0,0 +1,161 @@
+using BCG;
+using BCG.Utilities.General;
+
+using System;
+
+namespace bts {
+    [DataStreamUserRespresentation("tradebars")]
+    public class TradeBarsStream : ViewStream, IWorker {
+
+        public class TradeBar {
+            static double[] calculate_log_returns(double[] prices) {
+                // Calculate logarithmic returns
+                double[] returns = new double[prices.Length - 1];
+                for (int i = 1; i < prices.Length; i++) {
+                    returns[i - 1] = Math.Log(prices[i] / prices[i - 1]);
+                }
+                return returns;
+            }
+
+            static double calculate_volatility(double[] returns) {
+                // Calculate standard deviation of logarithmic returns
+                double mean = returns.Average();
+                double sum_of_squares = returns.Sum(x => Math.Pow(x - mean, 2));
+                double variance = sum_of_squares / (returns.Length - 1);
+                double volatility = Math.Sqrt(variance);
+
+                return volatility;
+            }
+            public void add_trade(IBlueRow r) {
+                double price = r.get<double>("price");
+                //prices.Add(price);
+                double qty = r.get<double>(col_name: "qty");
+                if (n_entries == 0) {
+                    open = price;
+                    low = price < low ? price : low;
+                    high = price > high ? price : high;
+                }
+                n_entries++;
+                volume += qty;
+                vwap_sum += price * qty;
+                close = price;
+                if ("Buy".Equals(r.get<string>("aggressor"))) {
+                    n_buys_aggs++;
+                } else {
+                    n_sell_aggs++;
+                }
+            }
+            private List<double> prices = new List<double>();
+            public readonly string security;
+            public double low = Double.MaxValue, high, open, close, median, std_dev, vwap_sum, volume;
+            public uint n_buys_aggs, n_sell_aggs, n_entries;
+            public DateTime bar_start = DateTime.Now;
+            public TradeBar(string security_) {
+                security = security_;
+            }
+            public double volatility {
+                get {
+                    //if (prices.Count == 1)
+                    return 0.0;
+                    var log_returns = calculate_log_returns(prices.ToArray());
+                    return calculate_volatility(log_returns);
+                }
+            }
+        }
+
+        private readonly int _bar_period_seconds;
+        private readonly ViewStreamDatabase _vsd;
+        private ViewStream _timesales_stream;
+        private ViewStream _bbo_stream;
+        private IDictionary<uint, TradeBar> _sec_bar_map = new Dictionary<uint, TradeBar>();
+        private bool _dirty = false;
+        private TimeSpan _fire_period;
+        private DateTime _last_fire_time = DateTime.Now;
+        private BlueTable _table = new BlueTable(BlueDataUtility.form_schema("TradeBars",
+        Tuple.Create("id", TypeCode.Int32),
+        Tuple.Create("bar_start", TypeCode.DateTime),
+        Tuple.Create("bar_end", TypeCode.DateTime),
+        Tuple.Create("day_of_week", TypeCode.String),
+        Tuple.Create("n_trades", TypeCode.UInt32),
+        Tuple.Create("security", TypeCode.String),
+        Tuple.Create("sec_key", TypeCode.UInt32),
+        Tuple.Create("low", TypeCode.Double),
+        Tuple.Create("high", TypeCode.Double),
+        Tuple.Create("open", TypeCode.Double),
+        Tuple.Create("close", TypeCode.Double),
+        Tuple.Create("vwap_sum", TypeCode.Double),
+        Tuple.Create("volume", TypeCode.Double),
+        Tuple.Create("n_buy_aggs", TypeCode.UInt32),
+        Tuple.Create("n_sell_aggs", TypeCode.UInt32),
+        Tuple.Create("volatility", TypeCode.Double)));
+
+
+        public TradeBarsStream(int bar_period_seconds)
+        : this(bar_period_seconds, ViewStreamDatabase.instance) { }
+        public TradeBarsStream(int bar_period_seconds, ViewStreamDatabase vsd) {
+            _vsd = vsd;
+            _bar_period_seconds = bar_period_seconds;
+            _timesales_stream = new TimeSalesStream(10000, vsd);
+            _fire_period = TimeSpan.FromSeconds(bar_period_seconds);
+            _bbo_stream = new BBOStream(_vsd);
+        }
+
+        protected override void myOnFirstSubscriber() {
+            _timesales_stream.onChange += _on_timesales;
+            _bbo_stream.onChange += _on_bbo;
+            _vsd.addWorker(this);
+        }
+
+        protected override void myOnLastSubscriber() {
+            _vsd.removeWorker(this);
+            _timesales_stream.onChange -= _on_timesales;
+            _bbo_stream.onChange -= _on_bbo;
+        }
+
+        public void doWork() {
+            if (DateTime.Now - _last_fire_time > _fire_period) {
+                if (_dirty) {
+                    _dirty = false;
+                    _last_fire_time = DateTime.Now;
+                    BlueTable added_rows = new BlueTable(_table.schema);
+                    foreach (var kv in _sec_bar_map) {
+                        var sec_key = kv.Key;
+                        var tbar = kv.Value;
+                        IBlueRow new_row = new BlueRow(_table.schema, _table.Count, tbar.bar_start, DateTime.Now,
+                        tbar.bar_start.DayOfWeek.ToString(),
+                                                        tbar.n_entries, tbar.security, sec_key, tbar.low, tbar.high, tbar.open,
+                                                        tbar.close, tbar.vwap_sum, tbar.volume, tbar.n_sell_aggs, tbar.n_sell_aggs,
+                                                        tbar.volatility);
+                        added_rows.addRow(new_row);
+                        _table.addRow(new_row);
+                    }
+                    _sec_bar_map.Clear();
+                    DataPacket dp_out = new DataPacket(_table, added_rows, new BlueTable(_table.schema));
+                    myFireEvent(dp_out);
+                }
+            }
+        }
+
+        private void _on_timesales(object obj) {
+            DataPacket dp = obj as DataPacket;
+            foreach (IBlueRow r in dp.addedRows) {
+                uint sec_key = r.get<UInt32>("sec_key");
+                TradeBar tbar = null;
+                if (!_sec_bar_map.TryGetValue(sec_key, out tbar)) {
+                    tbar = new TradeBar(r.get<string>("sec"));
+                    _sec_bar_map.Add(sec_key, tbar);
+                }
+                _dirty = true;
+                tbar.add_trade(r);
+            }
+
+
+        }
+
+        private void _on_bbo(object obj) {
+
+        }
+    }
+
+
+}

src/DataStreamSuite/SharedStreams/Business/MarketData/PriceStatsStream.cs

@@ -40,13 +40,16 @@ namespace BCG {
       IReadOnlyDictionary<SecurityKey, PriceStats> _cached_state = null;
       MarketDataClient _md_client;
       private bool _dirty = false;
+      private IDisposable _sub = null;
       private readonly IBlueTableSchema _pstats_scheam = BlueDataUtility.form_schema("bbo",
                                                 Tuple.Create("id", TypeCode.Int32),
-                                                Tuple.Create("sec_key", TypeCode.Int32),
+                                                Tuple.Create("sec_key", TypeCode.UInt32),
                                                 Tuple.Create("sec", TypeCode.String),
-                                                Tuple.Create("open", TypeCode.UInt32),
+                                                Tuple.Create("date", TypeCode.DateTime),
+                                                Tuple.Create("open", TypeCode.Double),
                                                 Tuple.Create("close", TypeCode.Double),
-                                                Tuple.Create("settlement", TypeCode.Single));
+                                                Tuple.Create("settlement", TypeCode.Double));
+      IDisposable? subscription;
 
       // TODO: add cons for symbol, exchange etc 
       public PriceStatsStream()
@@ -61,7 +64,7 @@ namespace BCG {
 
       protected override void myOnFirstSubscriber() {
          _vsd.addWorker(this);
-         _md_client.price_stats().Subscribe(_on_price_stats);
+         subscription = _md_client.price_stats().Subscribe(_on_price_stats);
       }
 
       private void _log_updates(TableUpdate<PriceStats, uint> update) {
@@ -80,11 +83,12 @@ namespace BCG {
          // just cacching in here, will populate the rows in _poll_scan and fire the datapacket
          bts.Log.Gen.info($"PriceStatsStream._on_price_stats state: {update.state.Count()} changes: {update.changes.Count()} added: {update.added.Count()}");
          _cached_state = update.state;
-         _dirty = false;
+         _dirty = true;
       }
 
       protected override void myOnLastSubscriber() {
          _vsd.removeWorker(this);
+         Disposable.dispose(ref _sub);
       }
 
       private void _scan() {
@@ -97,7 +101,8 @@ namespace BCG {
             SecurityKey skey = kv.Key;
             int id = Convert.ToInt32(skey);
             PriceStats pstats = kv.Value;
-            IBlueRow r = new BlueRow(_pstats_scheam, id, skey, pstats.security.ToString(), pstats.open, pstats.close, pstats.settlement);
+            var today = DateTime.Now.Date;
+            IBlueRow r = new BlueRow(_pstats_scheam, id, skey, pstats.security.ToString(), today, pstats.open, pstats.close, pstats.settlement);
             new_state.addRow(r);
          }
          DataPacket dp = null;
@@ -122,4 +127,4 @@ namespace BCG {
          }
       }
    }
-}
+}

src/DataStreamSuite/SharedStreams/SharedStreams.csproj

@@ -12,6 +12,7 @@
     <ProjectReference Include="..\DataStreamCore\DataStreamCore.csproj" />
     <ProjectReference Include="..\DataStream\DataStream.csproj" />
     <ProjectReference Include="..\PubSub\PubSub.csproj" />
+    <ProjectReference Include="..\..\client\spark_core\spark_core.csproj" />
   </ItemGroup>
 
   <ItemGroup>

src/Spark.Admin.WebApp/src/Accounts/Types.fs

@@ -17,9 +17,9 @@ type Tab =
   | PositionManagementTab
   | MarginTab
 
-let venues = [| Venue.DERIBIT; Venue.GLBX; Venue.ICE; Venue.SIM; |]
+let venues = [| Venue.BINANCE; Venue.BinanceUs; Venue.BinanceUsdsFutures; Venue.COINBASE; Venue.DERIBIT; Venue.GLBX; Venue.ICE; Venue.SIM; |]
 
-let securityTypes = [| SecurityType.FUTURE; SecurityType.OPTION; SecurityType.SPREAD; |]
+let securityTypes = [| SecurityType.SPOT; SecurityType.FUTURE; SecurityType.OPTION; SecurityType.SPREAD; |]
 
 let emptyAccount = Access.Accounts.empty
 
@@ -35,6 +35,26 @@ type CMEExchangeField =
   | ClearingAccount of string
   | MiscTagMapping of JSMap<string>
 
+type BINANCEExchangeField =
+  | CTI of CTICode
+  | Origin of CustomerOrFirm
+  | ClearingAccount of string
+
+type BinanceUsExchangeField =
+  | CTI of CTICode
+  | Origin of CustomerOrFirm
+  | ClearingAccount of string
+
+type BinanceUsdsFUTURESExchangeField =
+  | CTI of CTICode
+  | Origin of CustomerOrFirm
+  | ClearingAccount of string
+
+type COINBASEExchangeField =
+  | CTI of CTICode
+  | Origin of CustomerOrFirm
+  | ClearingAccount of string
+
 type DERIBITExchangeField =
   | CTI of CTICode
   | Origin of CustomerOrFirm

src/Spark.Admin.WebApp/src/Accounts/View.fs

@@ -797,6 +797,7 @@ let venueToString (venue:bts.api.Venue) =
     | bts.api.Venue.BINANCE -> "BINANCE"
     | bts.api.Venue.BinanceUs -> "BinanceUs"
     | bts.api.Venue.BinanceUsdsFutures -> "BinanceUsdsFutures"
+    | bts.api.Venue.COINBASE -> "COINBASE"
     | bts.api.Venue.FTX -> "FTX"
     | bts.api.Venue.OKX -> "OKX"
 

src/client/spark/Modules/Trading/AlgoStudio/Model/AlgoDataVM.cs

@@ -1,39 +1,42 @@
-using bts.api.orderman;
-using bts.api.orderman.algo;
-using bts.utils;
-
-using mbts.spark.utilities;
-
-using System;
-using System.Windows.Media;
-
-namespace mbts.spark {
-   struct AlgoDataVM : IEquatable<AlgoDataVM> {
-      public static readonly Func<AlgoDataVM, AlgoID> id_fn = vm => AlgoID.from_algo_key(vm.data.key);
-
-      public readonly AlgoData data;
-      public readonly AlgoDefinitionMetadata? meta;
-
-      public AlgoDataVM(AlgoData data, AlgoDefinitionMetadata? meta = null) {
-         this.data = data;
-         this.meta = meta;
-      }
-
-      public string name => data.name;
-      public Brush badge_fill => OrderUtils.badge_fill(meta, OrderDataHelper.get_metadata(data.value.CustomCase));
-      public AlgoDisplayGroup group => data.key.display_group();
-
-      public override string ToString() =>
-         data.name;
-
-      public static AlgoDataVM from_data(AlgoData data, AlgoDefinitionMetadata? meta) =>
-         new AlgoDataVM(data, meta);
-
-      public static AlgoDataVM from_data((AlgoData algo, AlgoDefinitionMetadata? meta) tup) =>
-         new AlgoDataVM(tup.algo, tup.meta);
-
-      public bool Equals(AlgoDataVM other) => data == other.data && meta == other.meta;
-      public override bool Equals(object obj) => obj is AlgoDataVM other && Equals(other);
-      public override int GetHashCode() => EqualityComparer.aggregate_hash(data.GetHashCode(), meta.GetHashCode());
-   }
-}
+using bts.api.orderman;
+using bts.api.orderman.algo;
+using bts.utils;
+
+using mbts.spark.utilities;
+
+using System;
+using System.Windows.Media;
+
+namespace mbts.spark {
+   struct AlgoDataVM : IEquatable<AlgoDataVM> {
+      public static readonly Func<AlgoDataVM, AlgoID> id_fn = vm => AlgoID.from_algo_key(vm.data.key);
+
+      public readonly AlgoData data;
+      public readonly AlgoDefinitionMetadata? meta;
+
+      public AlgoDataVM(AlgoData data, AlgoDefinitionMetadata? meta = null) {
+         this.data = data;
+         this.meta = meta;
+      }
+
+      public string name => data.name;
+
+      public Brush badge_fill => OrderUtils.badge_fill(
+         meta, OrderMetadataStore.get_metadata(data.value.CustomCase));
+
+      public AlgoDisplayGroup group => data.key.display_group();
+
+      public override string ToString() =>
+         data.name;
+
+      public static AlgoDataVM from_data(AlgoData data, AlgoDefinitionMetadata? meta) =>
+         new AlgoDataVM(data, meta);
+
+      public static AlgoDataVM from_data((AlgoData algo, AlgoDefinitionMetadata? meta) tup) =>
+         new AlgoDataVM(tup.algo, tup.meta);
+
+      public bool Equals(AlgoDataVM other) => data == other.data && meta == other.meta;
+      public override bool Equals(object obj) => obj is AlgoDataVM other && Equals(other);
+      public override int GetHashCode() => EqualityComparer.aggregate_hash(data.GetHashCode(), meta.GetHashCode());
+   }
+}

src/client/spark/Modules/Trading/OrderBook/OrderBookWindow.cs

@@ -298,7 +298,7 @@ namespace mbts.spark {
          }
 
          bool has_algo_params(OrderSecurityAlgo? order) =>
-            order.HasValue && !OrderDataHelper.get_metadata(order.Value.algo.value.CustomCase).NoProps;
+            order.HasValue && !OrderMetadataStore.get_metadata(order.Value.algo.value.CustomCase).NoProps;
 
          Window mk_algo_params_dialog(OrderSecurityAlgo? order) {
             if (!has_algo_params(order))

src/client/spark/Orders/OrderFactory.cs

@@ -1,323 +1,323 @@
-#nullable enable
-namespace mbts.spark.utilities {
-   using bts;
-   using bts.api;
-   using bts.api.core;
-   using bts.api.orderman;
-   using bts.collections;
-   using bts.spark;
-
-   using mbts.ui;
-   using mbts.ui.utilities;
-   using mbts.ui.utility;
-   using mbts.utils;
-
-   using Spark.Core;
-   using Spark.Core.Disposables;
-
-   using System;
-   using System.Collections.Generic;
-   using System.Composition;
-   using System.Linq;
-   using System.Reactive.Linq;
-
-   using OrderType = bts.api.orderman.CustomOrderDefinition.CustomOneofCase;
-   using DynamicOrderParamType = bts.api.orderman.DynamicOrderParam.ValueOneofCase;
-
-   [Export(typeof(OrderFactory))]
-   class OrderFactory : IDisposable {
-      static readonly OrderMetadata _default_order_metadata = new() {
-         KeyBinding = "",
-         Color = BTSColors.Orders.default_order
-      };
-      readonly AdminModel _admin_model;
-      readonly IReadOnlyDictionary<OneOf<OrderType, string>, string> _algo_abbreviation_index;
-      readonly IReadOnlyDictionary<string, DynamicOrderDescriptor> _dynamic_descriptors;
-
-      IDisposable? _dispose;
-
-      [ImportingConstructor]
-      public OrderFactory(OrderService service, AdminModel admin_model) {
-         _admin_model = admin_model;
-         _dynamic_descriptors = service.get_registered_dynamic_order_types(new VoidMessage())
-            .DynamicOrderDesc
-            .ToDictionary(d => d.DynamicOrderType);
-         var algo_name_index = new HashSet<OneOf<OrderType, string>>(
-            OneOf<OrderType, string>.ValueEqualityComparer);
-         // add standard types
-         algo_name_index.AddRange(
-            EnumHelper.get_values_except(OrderType.None)
-            .Select(t => new OneOf<OrderType, string>(t)));
-         // add dynamic types
-         algo_name_index.AddRange(_dynamic_descriptors.Keys
-            .Select(dt => new OneOf<OrderType, string>(dt)));
-         // build abbreviation index
-         _algo_abbreviation_index = DisplayNameHelper.get_word_tree_abbreviations(
-            algo_name_index,
-            oneof => oneof.value.ToString(),
-            OneOf<OrderType, string>.ValueEqualityComparer);
-      }
-
-      public static string? fmt_dynamic_param(DynamicOrderParam? dp) {
-         if (dp is null) {
-            return null;
-         }
-         return dp.ValueCase switch {
-            DynamicOrderParamType.AsArray => ", ".join(dp.AsArray.Element.Select(fmt_dynamic_param)),
-            DynamicOrderParamType.AsBool => dp.AsBool.ToString(),
-            DynamicOrderParamType.AsDouble => dp.AsDouble.ToString(),
-            DynamicOrderParamType.AsEnumerated => dp.AsEnumerated.EnumValue,
-            DynamicOrderParamType.AsInt32 => dp.AsInt32.ToString(),
-            DynamicOrderParamType.AsQuantity => dp.AsQuantity.ToString(),
-            DynamicOrderParamType.AsString => dp.AsString,
-            _ => $"({dp.ValueCase})",
-         };
-      }
-
-      public static DynamicOrderParam init_dynamic_param(DynamicOrderParamDescriptor descriptor) {
-         var rv = new DynamicOrderParam();
-         switch (descriptor.ParamType.DescriptorCase) {
-            case DynamicOrderTypeDescriptor.DescriptorOneofCase.Array:
-               rv.AsArray = new DynamicOrderArray();
-               break;
-            case DynamicOrderTypeDescriptor.DescriptorOneofCase.Enumerated:
-               rv.AsEnumerated = new() {
-                  EnumValue = descriptor.ParamType.Enumerated.Element.FirstOrDefault()
-               };
-               break;
-            case DynamicOrderTypeDescriptor.DescriptorOneofCase.Primitive:
-               switch (descriptor.ParamType.Primitive) {
-                  case DynamicOrderPrimitiveDescriptor.DynamicOrderPrimitiveBool: rv.AsBool = false; break;
-                  case DynamicOrderPrimitiveDescriptor.DynamicOrderPrimitiveDouble: rv.AsDouble = 0.0; break;
-                  case DynamicOrderPrimitiveDescriptor.DynamicOrderPrimitiveInt: rv.AsInt32 = 0; break;
-                  case DynamicOrderPrimitiveDescriptor.DynamicOrderPrimitiveOrderDef: rv.AsOrderDef = new(); break;
-                  case DynamicOrderPrimitiveDescriptor.DynamicOrderPrimitiveString: rv.AsString = string.Empty; break;
-                  case DynamicOrderPrimitiveDescriptor.DynamicOrderPrimitiveSecurityKey:
-                  case DynamicOrderPrimitiveDescriptor.DynamicOrderPrimitiveUint: rv.AsUint32 = 0u; break;
-                  case DynamicOrderPrimitiveDescriptor.DynamicOrderPrimitiveQuantity: rv.AsQuantity = new(); break;
-               }
-               break;
-            case DynamicOrderTypeDescriptor.DescriptorOneofCase.Structure:
-               rv.AsStruct = new();
-               break;
-         }
-         return rv;
-      }
-
-      public static SpreaderOrderDefinition spreader_def() => new() {
-         ExecutionStrategy = new ExecutionStrategy()
-      };
-
-      public DynamicOrderDescriptor? dynamic_descriptor(string type) =>
-         _dynamic_descriptors.GetValueOrDefault(type);
-
-      public DynamicOrderDefinition? dynamic_def(string type) {
-         if (!_dynamic_descriptors.TryGetValue(type, out var desc)) {
-            return null;
-         }
-         var def = new DynamicOrderDefinition { DynamicOrderType = type };
-         foreach (var p in desc.ParamDesc) {
-            def.OrderParams[p.ParamName] = init_dynamic_param(p);
-         }
-         return def;
-      }
-
-      public OrderType default_order_type(SecurityImmutable security) =>
-         security.ContractCase switch {
-            Security.ContractOneofCase.Synthetic => OrderType.Spreader,
-            _ => OrderType.Limit,
-         };
-
-      public TimeInForce default_time_in_force() =>
-         TimeInForce.Day;
-
-      public TimeInForce default_time_in_force(SecurityImmutable security) =>
-         default_time_in_force();
-
-      public IEnumerable<OrderType> default_order_types() =>
-         OrderDataHelper.exchange_order_types;
-
-      public IEnumerable<TimeInForce> default_time_in_forces(
-         SecurityImmutable security,
-         OrderType order_type,
-         bool is_child = false
-      ) {
-         var tifs = EnumHelper.get_values_except(TimeInForce.Undefined);
-         if (order_type != OrderType.Limit) {
-            tifs = tifs.Except(new[] { TimeInForce.Ioc });
-         }
-         return tifs;
-      }
-
-      public IEnumerable<string> dynamic_types() =>
-         _dynamic_descriptors.Keys;
-
-      public CustomOrderDefinitionImmutable? custom_defi(string type_name) {
-         if (Enum.TryParse(type_name, out OrderType sys_type)) {
-            return _custom_def(sys_type);
-         }
-         var dyn_def = dynamic_def(type_name);
-         if (dyn_def is null) {
-            return null;
-         }
-         return new(new() { Dynamic = dyn_def });
-      }
-
-      public OrderDefinition order(SecurityImmutable security, AccountTuple? account = null) {
-         var rv = new OrderDefinition {
-            SecurityKey = security.Id,
-            Side = Side.Buy,
-            TimeInForce = TimeInForce.Day,
-            CustomDef = OrderDataHelper.Defaults.CustomOrderDefinition(),
-            CtiCode = CTICode.CtiCode1,
-            CustomerOrFirm = CustomerOrFirm.Firm,
-            // this will be the tag 50 on CME orders
-            // current user might be okay, but we might also want to get it
-            // from the store that serves the risk admin gui
-            ManualOrderIndicator = ManualOrderIndicator.Yes,
-            User = _admin_model.user.name,
-            UserId = _admin_model.user.id
-         };
-         if (account.HasValue) {
-            rv.Account = account.Value.name;
-            rv.AccountId = account.Value.id;
-         }
-         return rv;
-      }
-
-      public string abbreviate_algo(OneOf<OrderType, string> id) {
-         if (id is null) {
-            throw new ArgumentNullException(nameof(id));
-         }
-         if (id.value is string) {
-            return _algo_abbreviation_index.GetValueOrDefault(id) ??
-               id.value.ToString() ??
-               string.Empty;
-         }
-         if (id.value is OrderType t) {
-            return OrderDataHelper.abbreviate_algo_type(t);
-         }
-         return id.ToString() ?? string.Empty;
-      }
-
-      public OrderMetadata get_metadata(OrderRecordImmutable order) =>
-         get_metadata(order.Def.CustomDef.CustomCase);
-
-      public OrderMetadata get_metadata(OrderType order_type) =>
-         order_type switch {
-            OrderType.Market => new() { KeyBinding = "M", Color = BTSColors.Orders.market_order, NoProps = true },
-            OrderType.Limit => new() { KeyBinding = "L", Color = BTSColors.Orders.limit_order, NoProps = true },
-            OrderType.Stop => new() { KeyBinding = "S", Color = BTSColors.Orders.stop_order, NoProps = true },
-            OrderType.StopLimit => new() { KeyBinding = "SL", Color = BTSColors.Orders.stop_limit_order },
-            OrderType.GlbxIceberg => new() { KeyBinding = "GB", Color = BTSColors.Orders.glbx_iceberg_order },
-            OrderType.Iceberg => new() { KeyBinding = "I", Color = BTSColors.Orders.iceberg_order },
-            OrderType.TimeBox => new() { KeyBinding = "TB", Color = BTSColors.Orders.time_box_order },
-            OrderType.Tick => new() { KeyBinding = "T", Color = BTSColors.Orders.tick_order },
-            OrderType.Lean => new() { KeyBinding = "L", Color = BTSColors.Orders.lean_order },
-            OrderType.Join => new() { KeyBinding = "J", Color = BTSColors.Orders.join_order },
-            OrderType.Top => new() { KeyBinding = "D", Color = BTSColors.Orders.top_order },
-            OrderType.Cover => new() { KeyBinding = "C", Color = BTSColors.Orders.cover_order },
-            OrderType.Oco => new() { KeyBinding = "C", Color = BTSColors.Orders.oco_order },
-            OrderType.Pricer => new() { KeyBinding = "C", Color = BTSColors.Orders.pricer_order },
-            OrderType.Offset => new() { KeyBinding = "O", Color = BTSColors.Orders.off_order },
-            OrderType.Trigger => new() { KeyBinding = "T", Color = BTSColors.Orders.trigger_order },
-            OrderType.Spreader => new() { KeyBinding = "", Color = BTSColors.Orders.spreader_order },
-            OrderType.Dynamic => new() { KeyBinding = "", Color = BTSColors.Orders.top_order },
-            _ => _default_order_metadata
-         };
-
-      public SpreaderOrderDefinition spreader_order(
-         SecurityImmutable security, ExecutionStrategyImmutable strategy) =>
-         new() {
-            Security = security.CloneToMutable(),
-            ExecutionStrategy = strategy.CloneToMutable()
-         };
-
-      public SpreaderOrderDefinitionImmutable spreader_orderi(
-         SecurityImmutable security, ExecutionStrategyImmutable strategy) =>
-         new(spreader_order(security, strategy));
-
-      public void Dispose() =>
-         Disposable.dispose(ref _dispose);
-
-      static CustomOrderDefinition _custom_def() =>
-         new() { Limit = _limit_def() };
-
-      static CustomOrderDefinition _custom_def(
-         OrderType type) {
-         var rv = new CustomOrderDefinition();
-         switch (type) {
-            case OrderType.Market: rv.Market = _market_def(); break;
-            case OrderType.Limit: rv.Limit = _limit_def(); break;
-            case OrderType.Stop: rv.Stop = _stop_def(); break;
-            case OrderType.StopLimit: rv.StopLimit = _stop_limit_def(); break;
-            case OrderType.Iceberg: rv.Iceberg = _iceberg_def(); break;
-            case OrderType.GlbxIceberg: rv.GlbxIceberg = _glbx_iceberg_def(); break;
-            case OrderType.TimeBox: rv.TimeBox = _time_box_def(); break;
-            case OrderType.Tick: rv.Tick = _tick_def(); break;
-            case OrderType.Lean: rv.Lean = _lean_def(); break;
-            case OrderType.Join: rv.Join = _join_def(); break;
-            case OrderType.Top: rv.Top = _top_def(); break;
-            case OrderType.Cover: rv.Cover = _cover_def(); break;
-            case OrderType.Oco: rv.Oco = _oco_def(); break;
-            case OrderType.Pricer: rv.Pricer = _pricer_def(); break;
-            case OrderType.Offset: rv.Offset = _offset_def(); break;
-            case OrderType.Trigger: rv.Trigger = _trigger_def(); break;
-            case OrderType.Spreader: rv.Spreader = spreader_def(); break;
-            case OrderType.Dynamic: rv.Dynamic = _dynamic_def(); break;
-            default:
-               OrderTypeVMFactory.GenericOrderFactory.setup_generic_custom_order(rv, type);
-               break;
-         }
-         return rv;
-      }
-
-      static MarketOrderDefinition _market_def() => new();
-      static LimitOrderDefinition _limit_def() => new();
-      static StopOrderDefinition _stop_def() => new();
-      static StopLimitOrderDefinition _stop_limit_def() => new();
-      static GLBXIcebergOrderDefinition _glbx_iceberg_def() => new() { MaxShowQuantity = 1 };
-      static IcebergOrderDefinition _iceberg_def() =>
-         new() { Child = _custom_def(), ShowQuantity = 1 };
-      static TimeBoxOrderDefinition _time_box_def() => new() {
-         Child = _custom_def(),
-         Start = Google.Protobuf.WellKnownTypes.Timestamp.FromDateTime(DateTime.UtcNow),
-         Stop = Google.Protobuf.WellKnownTypes.Timestamp.FromDateTime(DateTime.UtcNow)
-      };
-      static TickOrderDefinition _tick_def() => new() {
-         Child = _custom_def(),
-         CrossThreshold = new() { Absolute = 1 }
-      };
-      static LeanOrderDefinition _lean_def() => new() {
-         Child = _custom_def(),
-         CrossThreshold = new() { Absolute = 1 },
-         MaxChaseTicks = 2
-      };
-      static JoinOrderDefinition _join_def() => new() {
-         Child = _custom_def(),
-         JoinThreshold = new() { Absolute = 1 }
-      };
-      static TopOrderDefinition _top_def() => new() {
-         Child = _custom_def(),
-         TopThreshold = new() { Absolute = 1 }
-      };
-      static CoverOrderDefinition _cover_def() => new() {
-         Child = _custom_def(),
-         Hedge = _custom_def(),
-         HedgeTicks = 1
-      };
-      static OCOOrderDefinition _oco_def() => new() {
-         Child = _custom_def(),
-         Other = _custom_def(),
-         OtherSide = RelativeSide.Same
-      };
-      static PricerOrderDefinition _pricer_def() => new() { Child = _custom_def() };
-      static OffsetOrderDefinition _offset_def() => new() { Child = _custom_def(), };
-      static TriggerOrderDefinition _trigger_def() => new() {
-         Child = _custom_def(),
-         TriggerCondition = new TriggerCondition(),
-         TriggerAction = new TriggerAction(),
-      };
-      static DynamicOrderDefinition _dynamic_def() => new();
-   }
-}
+#nullable enable
+namespace mbts.spark.utilities {
+   using bts;
+   using bts.api;
+   using bts.api.core;
+   using bts.api.orderman;
+   using bts.collections;
+   using bts.spark;
+
+   using mbts.ui;
+   using mbts.ui.utilities;
+   using mbts.ui.utility;
+   using mbts.utils;
+
+   using Spark.Core;
+   using Spark.Core.Disposables;
+
+   using System;
+   using System.Collections.Generic;
+   using System.Composition;
+   using System.Linq;
+   using System.Reactive.Linq;
+
+   using OrderType = bts.api.orderman.CustomOrderDefinition.CustomOneofCase;
+   using DynamicOrderParamType = bts.api.orderman.DynamicOrderParam.ValueOneofCase;
+
+   [Export(typeof(OrderFactory))]
+   class OrderFactory : IDisposable {
+      static readonly OrderMetadata _default_order_metadata = new() {
+         KeyBinding = "",
+         Color = BTSColors.Orders.default_order
+      };
+      readonly AdminModel _admin_model;
+      readonly IReadOnlyDictionary<OneOf<OrderType, string>, string> _algo_abbreviation_index;
+      readonly IReadOnlyDictionary<string, DynamicOrderDescriptor> _dynamic_descriptors;
+
+      IDisposable? _dispose;
+
+      [ImportingConstructor]
+      public OrderFactory(OrderService service, AdminModel admin_model) {
+         _admin_model = admin_model;
+         _dynamic_descriptors = service.get_registered_dynamic_order_types(new VoidMessage())
+            .DynamicOrderDesc
+            .ToDictionary(d => d.DynamicOrderType);
+         var algo_name_index = new HashSet<OneOf<OrderType, string>>(
+            OneOf<OrderType, string>.ValueEqualityComparer);
+         // add standard types
+         algo_name_index.AddRange(
+            EnumHelper.get_values_except(OrderType.None)
+            .Select(t => new OneOf<OrderType, string>(t)));
+         // add dynamic types
+         algo_name_index.AddRange(_dynamic_descriptors.Keys
+            .Select(dt => new OneOf<OrderType, string>(dt)));
+         // build abbreviation index
+         _algo_abbreviation_index = DisplayNameHelper.get_word_tree_abbreviations(
+            algo_name_index,
+            oneof => oneof.value.ToString(),
+            OneOf<OrderType, string>.ValueEqualityComparer);
+      }
+
+      public static string? fmt_dynamic_param(DynamicOrderParam? dp) {
+         if (dp is null) {
+            return null;
+         }
+         return dp.ValueCase switch {
+            DynamicOrderParamType.AsArray => ", ".join(dp.AsArray.Element.Select(fmt_dynamic_param)),
+            DynamicOrderParamType.AsBool => dp.AsBool.ToString(),
+            DynamicOrderParamType.AsDouble => dp.AsDouble.ToString(),
+            DynamicOrderParamType.AsEnumerated => dp.AsEnumerated.EnumValue,
+            DynamicOrderParamType.AsInt32 => dp.AsInt32.ToString(),
+            DynamicOrderParamType.AsQuantity => dp.AsQuantity.ToString(),
+            DynamicOrderParamType.AsString => dp.AsString,
+            _ => $"({dp.ValueCase})",
+         };
+      }
+
+      public static DynamicOrderParam init_dynamic_param(DynamicOrderParamDescriptor descriptor) {
+         var rv = new DynamicOrderParam();
+         switch (descriptor.ParamType.DescriptorCase) {
+            case DynamicOrderTypeDescriptor.DescriptorOneofCase.Array:
+               rv.AsArray = new DynamicOrderArray();
+               break;
+            case DynamicOrderTypeDescriptor.DescriptorOneofCase.Enumerated:
+               rv.AsEnumerated = new() {
+                  EnumValue = descriptor.ParamType.Enumerated.Element.FirstOrDefault()
+               };
+               break;
+            case DynamicOrderTypeDescriptor.DescriptorOneofCase.Primitive:
+               switch (descriptor.ParamType.Primitive) {
+                  case DynamicOrderPrimitiveDescriptor.DynamicOrderPrimitiveBool: rv.AsBool = false; break;
+                  case DynamicOrderPrimitiveDescriptor.DynamicOrderPrimitiveDouble: rv.AsDouble = 0.0; break;
+                  case DynamicOrderPrimitiveDescriptor.DynamicOrderPrimitiveInt: rv.AsInt32 = 0; break;
+                  case DynamicOrderPrimitiveDescriptor.DynamicOrderPrimitiveOrderDef: rv.AsOrderDef = new(); break;
+                  case DynamicOrderPrimitiveDescriptor.DynamicOrderPrimitiveString: rv.AsString = string.Empty; break;
+                  case DynamicOrderPrimitiveDescriptor.DynamicOrderPrimitiveSecurityKey:
+                  case DynamicOrderPrimitiveDescriptor.DynamicOrderPrimitiveUint: rv.AsUint32 = 0u; break;
+                  case DynamicOrderPrimitiveDescriptor.DynamicOrderPrimitiveQuantity: rv.AsQuantity = new(); break;
+               }
+               break;
+            case DynamicOrderTypeDescriptor.DescriptorOneofCase.Structure:
+               rv.AsStruct = new();
+               break;
+         }
+         return rv;
+      }
+
+      public static SpreaderOrderDefinition spreader_def() => new() {
+         ExecutionStrategy = new ExecutionStrategy()
+      };
+
+      public DynamicOrderDescriptor? dynamic_descriptor(string type) =>
+         _dynamic_descriptors.GetValueOrDefault(type);
+
+      public DynamicOrderDefinition? dynamic_def(string type) {
+         if (!_dynamic_descriptors.TryGetValue(type, out var desc)) {
+            return null;
+         }
+         var def = new DynamicOrderDefinition { DynamicOrderType = type };
+         foreach (var p in desc.ParamDesc) {
+            def.OrderParams[p.ParamName] = init_dynamic_param(p);
+         }
+         return def;
+      }
+
+      public OrderType default_order_type(SecurityImmutable security) =>
+         security.ContractCase switch {
+            Security.ContractOneofCase.Synthetic => OrderType.Spreader,
+            _ => OrderType.Limit,
+         };
+
+      public TimeInForce default_time_in_force() =>
+         TimeInForce.Day;
+
+      public TimeInForce default_time_in_force(SecurityImmutable security) =>
+         default_time_in_force();
+
+      public IEnumerable<OrderType> default_order_types() =>
+         OrderDataHelper.exchange_order_types;
+
+      public IEnumerable<TimeInForce> default_time_in_forces(
+         SecurityImmutable security,
+         OrderType order_type,
+         bool is_child = false
+      ) {
+         var tifs = EnumHelper.get_values_except(TimeInForce.Undefined);
+         if (order_type != OrderType.Limit) {
+            tifs = tifs.Except(new[] { TimeInForce.Ioc });
+         }
+         return tifs;
+      }
+
+      public IEnumerable<string> dynamic_types() =>
+         _dynamic_descriptors.Keys;
+
+      public CustomOrderDefinitionImmutable? custom_defi(string type_name) {
+         if (Enum.TryParse(type_name, out OrderType sys_type)) {
+            return _custom_def(sys_type);
+         }
+         var dyn_def = dynamic_def(type_name);
+         if (dyn_def is null) {
+            return null;
+         }
+         return new(new() { Dynamic = dyn_def });
+      }
+
+      public OrderDefinition order(SecurityImmutable security, AccountTuple? account = null) {
+         var rv = new OrderDefinition {
+            SecurityKey = security.Id,
+            Side = Side.Buy,
+            TimeInForce = TimeInForce.Day,
+            CustomDef = OrderDataHelper.Defaults.CustomOrderDefinition(),
+            CtiCode = CTICode.CtiCode1,
+            CustomerOrFirm = CustomerOrFirm.Firm,
+            // this will be the tag 50 on CME orders
+            // current user might be okay, but we might also want to get it
+            // from the store that serves the risk admin gui
+            ManualOrderIndicator = ManualOrderIndicator.Yes,
+            User = _admin_model.user.name,
+            UserId = _admin_model.user.id
+         };
+         if (account.HasValue) {
+            rv.Account = account.Value.name;
+            rv.AccountId = account.Value.id;
+         }
+         return rv;
+      }
+
+      public string abbreviate_algo(OneOf<OrderType, string> id) {
+         if (id is null) {
+            throw new ArgumentNullException(nameof(id));
+         }
+         if (id.value is string) {
+            return _algo_abbreviation_index.GetValueOrDefault(id) ??
+               id.value.ToString() ??
+               string.Empty;
+         }
+         if (id.value is OrderType t) {
+            return OrderDataHelper.abbreviate_algo_type(t);
+         }
+         return id.ToString() ?? string.Empty;
+      }
+
+      public OrderMetadata get_metadata(OrderRecordImmutable order) =>
+         get_metadata(order.Def.CustomDef.CustomCase);
+
+      public OrderMetadata get_metadata(OrderType order_type) =>
+         order_type switch {
+            OrderType.Market => new() { KeyBinding = "M", Color = BTSColors.Orders.market_order, NoProps = true },
+            OrderType.Limit => new() { KeyBinding = "L", Color = BTSColors.Orders.limit_order, NoProps = true },
+            OrderType.Stop => new() { KeyBinding = "S", Color = BTSColors.Orders.stop_order, NoProps = true },
+            OrderType.StopLimit => new() { KeyBinding = "SL", Color = BTSColors.Orders.stop_limit_order },
+            OrderType.GlbxIceberg => new() { KeyBinding = "GB", Color = BTSColors.Orders.glbx_iceberg_order },
+            OrderType.Iceberg => new() { KeyBinding = "I", Color = BTSColors.Orders.iceberg_order },
+            OrderType.TimeBox => new() { KeyBinding = "TB", Color = BTSColors.Orders.time_box_order },
+            OrderType.Tick => new() { KeyBinding = "T", Color = BTSColors.Orders.tick_order },
+            OrderType.Lean => new() { KeyBinding = "L", Color = BTSColors.Orders.lean_order },
+            OrderType.Join => new() { KeyBinding = "J", Color = BTSColors.Orders.join_order },
+            OrderType.Top => new() { KeyBinding = "D", Color = BTSColors.Orders.top_order },
+            OrderType.Cover => new() { KeyBinding = "C", Color = BTSColors.Orders.cover_order },
+            OrderType.Oco => new() { KeyBinding = "C", Color = BTSColors.Orders.oco_order },
+            OrderType.Pricer => new() { KeyBinding = "C", Color = BTSColors.Orders.pricer_order },
+            OrderType.Offset => new() { KeyBinding = "O", Color = BTSColors.Orders.off_order },
+            OrderType.Trigger => new() { KeyBinding = "T", Color = BTSColors.Orders.trigger_order },
+            OrderType.Spreader => new() { KeyBinding = "", Color = BTSColors.Orders.spreader_order },
+            OrderType.Dynamic => new() { KeyBinding = "", Color = BTSColors.Orders.top_order },
+            _ => _default_order_metadata
+         };
+
+      public SpreaderOrderDefinition spreader_order(
+         SecurityImmutable security, ExecutionStrategyImmutable strategy) =>
+         new() {
+            Security = security.CloneToMutable(),
+            ExecutionStrategy = strategy.CloneToMutable()
+         };
+
+      public SpreaderOrderDefinitionImmutable spreader_orderi(
+         SecurityImmutable security, ExecutionStrategyImmutable strategy) =>
+         new(spreader_order(security, strategy));
+
+      public void Dispose() =>
+         Disposable.dispose(ref _dispose);
+
+      static CustomOrderDefinition _custom_def() =>
+         new() { Limit = _limit_def() };
+
+      static CustomOrderDefinition _custom_def(
+         OrderType type) {
+         var rv = new CustomOrderDefinition();
+         switch (type) {
+            case OrderType.Market: rv.Market = _market_def(); break;
+            case OrderType.Limit: rv.Limit = _limit_def(); break;
+            case OrderType.Stop: rv.Stop = _stop_def(); break;
+            case OrderType.StopLimit: rv.StopLimit = _stop_limit_def(); break;
+            case OrderType.Iceberg: rv.Iceberg = _iceberg_def(); break;
+            case OrderType.GlbxIceberg: rv.GlbxIceberg = _glbx_iceberg_def(); break;
+            case OrderType.TimeBox: rv.TimeBox = _time_box_def(); break;
+            case OrderType.Tick: rv.Tick = _tick_def(); break;
+            case OrderType.Lean: rv.Lean = _lean_def(); break;
+            case OrderType.Join: rv.Join = _join_def(); break;
+            case OrderType.Top: rv.Top = _top_def(); break;
+            case OrderType.Cover: rv.Cover = _cover_def(); break;
+            case OrderType.Oco: rv.Oco = _oco_def(); break;
+            case OrderType.Pricer: rv.Pricer = _pricer_def(); break;
+            case OrderType.Offset: rv.Offset = _offset_def(); break;
+            case OrderType.Trigger: rv.Trigger = _trigger_def(); break;
+            case OrderType.Spreader: rv.Spreader = spreader_def(); break;
+            case OrderType.Dynamic: rv.Dynamic = _dynamic_def(); break;
+            default:
+               OrderDataHelper.setup_generic_custom_order(rv, type);
+               break;
+         }
+         return rv;
+      }
+
+      static MarketOrderDefinition _market_def() => new();
+      static LimitOrderDefinition _limit_def() => new();
+      static StopOrderDefinition _stop_def() => new();
+      static StopLimitOrderDefinition _stop_limit_def() => new();
+      static GLBXIcebergOrderDefinition _glbx_iceberg_def() => new() { MaxShowQuantity = 1 };
+      static IcebergOrderDefinition _iceberg_def() =>
+         new() { Child = _custom_def(), ShowQuantity = 1 };
+      static TimeBoxOrderDefinition _time_box_def() => new() {
+         Child = _custom_def(),
+         Start = Google.Protobuf.WellKnownTypes.Timestamp.FromDateTime(DateTime.UtcNow),
+         Stop = Google.Protobuf.WellKnownTypes.Timestamp.FromDateTime(DateTime.UtcNow)
+      };
+      static TickOrderDefinition _tick_def() => new() {
+         Child = _custom_def(),
+         CrossThreshold = new() { Absolute = 1 }
+      };
+      static LeanOrderDefinition _lean_def() => new() {
+         Child = _custom_def(),
+         CrossThreshold = new() { Absolute = 1 },
+         MaxChaseTicks = 2
+      };
+      static JoinOrderDefinition _join_def() => new() {
+         Child = _custom_def(),
+         JoinThreshold = new() { Absolute = 1 }
+      };
+      static TopOrderDefinition _top_def() => new() {
+         Child = _custom_def(),
+         TopThreshold = new() { Absolute = 1 }
+      };
+      static CoverOrderDefinition _cover_def() => new() {
+         Child = _custom_def(),
+         Hedge = _custom_def(),
+         HedgeTicks = 1
+      };
+      static OCOOrderDefinition _oco_def() => new() {
+         Child = _custom_def(),
+         Other = _custom_def(),
+         OtherSide = RelativeSide.Same
+      };
+      static PricerOrderDefinition _pricer_def() => new() { Child = _custom_def() };
+      static OffsetOrderDefinition _offset_def() => new() { Child = _custom_def(), };
+      static TriggerOrderDefinition _trigger_def() => new() {
+         Child = _custom_def(),
+         TriggerCondition = new TriggerCondition(),
+         TriggerAction = new TriggerAction(),
+      };
+      static DynamicOrderDefinition _dynamic_def() => new();
+   }
+}

src/client/spark/Orders/OrderTypeVMs/DefaultAlgoVM.cs

@@ -35,7 +35,7 @@ namespace mbts.spark {
       public IAlgoVM impl { get; private set; }
       public AlgoKey id { get; } = AlgoKey.from_known_type(AlgoType.None);
       public AlgoType order_type => _definition.value.CustomCase;
-      public OrderMetadata meta => OrderDataHelper.get_metadata(order_type);
+      public OrderMetadata meta => OrderMetadataStore.get_metadata(order_type);
       public bool can_expand => impl?.can_expand ?? false;
       public bool is_expanded {
          get => false;

src/client/spark/Orders/OrderTypeVMs/GenericOrderVM.cs

@@ -41,17 +41,6 @@ namespace mbts.spark {
             object vm = Activator.CreateInstance(makevm, key, algo_type, display_name, editing_permissions);
             return (IAlgoVM)vm;
          }
-         static public void setup_generic_custom_order(CustomOrderDefinition custom, CustomOrderDefinition.CustomOneofCase type) {
-            var prop = typeof(CustomOrderDefinition).GetProperty(type.ToString(),
-               BindingFlags.Instance | BindingFlags.Public);
-            var rt = prop.PropertyType;
-            var rtt = Activator.CreateInstance(rt);
-
-            var ci = rt.GetProperties().Where(x => x.PropertyType == typeof(CustomOrderDefinition));
-            foreach (var cid in ci)
-               cid.SetValue(rtt, new CustomOrderDefinition { Limit = new LimitOrderDefinition() });
-            prop.SetValue(custom, rtt);
-         }
       }
 
       class GenericOrderVM<TMutable, TImmutable> : OrderTypeVMBase<TMutable, TImmutable>

src/client/spark/Orders/OrderTypeVMs/OrderMetadataStore.cs

@@ -0,0 +1,32 @@
+namespace mbts.spark;
+
+using bts.api.orderman;
+using bts.spark;
+
+using mbts.ui;
+
+static class OrderMetadataStore {
+   public static OrderMetadata get_metadata(CustomOrderDefinition.CustomOneofCase order_type) =>
+      order_type switch {
+         CustomOrderDefinition.CustomOneofCase.Market => new() { KeyBinding = "M", Color = BTSColors.Orders.market_order, NoProps = true },
+         CustomOrderDefinition.CustomOneofCase.Limit => new() { KeyBinding = "L", Color = BTSColors.Orders.limit_order, NoProps = true },
+         CustomOrderDefinition.CustomOneofCase.Stop => new() { KeyBinding = "S", Color = BTSColors.Orders.stop_order, NoProps = true },
+         CustomOrderDefinition.CustomOneofCase.StopLimit => new() { KeyBinding = "SL", Color = BTSColors.Orders.stop_limit_order },
+         CustomOrderDefinition.CustomOneofCase.GlbxIceberg => new() { KeyBinding = "GB", Color = BTSColors.Orders.glbx_iceberg_order },
+         CustomOrderDefinition.CustomOneofCase.Iceberg => new() { KeyBinding = "I", Color = BTSColors.Orders.iceberg_order },
+         CustomOrderDefinition.CustomOneofCase.TimeBox => new() { KeyBinding = "TB", Color = BTSColors.Orders.time_box_order },
+         CustomOrderDefinition.CustomOneofCase.Tick => new() { KeyBinding = "T", Color = BTSColors.Orders.tick_order },
+         CustomOrderDefinition.CustomOneofCase.Lean => new() { KeyBinding = "L", Color = BTSColors.Orders.lean_order },
+         CustomOrderDefinition.CustomOneofCase.Join => new() { KeyBinding = "J", Color = BTSColors.Orders.join_order },
+         CustomOrderDefinition.CustomOneofCase.Top => new() { KeyBinding = "D", Color = BTSColors.Orders.top_order },
+         CustomOrderDefinition.CustomOneofCase.Cover => new() { KeyBinding = "C", Color = BTSColors.Orders.cover_order },
+         CustomOrderDefinition.CustomOneofCase.Oco => new() { KeyBinding = "C", Color = BTSColors.Orders.oco_order },
+         CustomOrderDefinition.CustomOneofCase.Pricer => new() { KeyBinding = "C", Color = BTSColors.Orders.pricer_order },
+         CustomOrderDefinition.CustomOneofCase.Offset => new() { KeyBinding = "O", Color = BTSColors.Orders.off_order },
+         CustomOrderDefinition.CustomOneofCase.Trigger => new() { KeyBinding = "T", Color = BTSColors.Orders.trigger_order },
+         CustomOrderDefinition.CustomOneofCase.Spreader => new() { KeyBinding = "", Color = BTSColors.Orders.spreader_order },
+         CustomOrderDefinition.CustomOneofCase.Dynamic => new() { KeyBinding = "", Color = BTSColors.Orders.dynamic_order },
+         _ => new OrderMetadata { KeyBinding = "", Color = BTSColors.Orders.default_order },
+      };
+
+}

src/client/spark/Orders/OrderTypeVMs/OrderTypeVMFactory.cs

@@ -201,7 +201,7 @@ namespace mbts.spark {
          public virtual string abbreviation => model == null ? display_name : model.factory.abbreviate_algo(new OneOf<AlgoType, string>(order_type));
          public virtual AlgoDisplayGroup group => order_type.is_bts_algo_type() ? AlgoDisplayGroup.BTS : AlgoDisplayGroup.Exchange;
          public OrdersModel orders_model => model;
-         public virtual OrderMetadata meta => OrderDataHelper.get_metadata(order_type);
+         public virtual OrderMetadata meta => OrderMetadataStore.get_metadata(order_type);
          // child algos
          [EditorBrowsable(EditorBrowsableState.Never)]
          public dynamic children_vm {

src/client/spark_api/MarketData/BookTuple.cs

@@ -36,7 +36,7 @@ namespace Spark.Api.MarketData {
       public override int GetHashCode() =>
          security.Id.GetHashCode();
       public static Object ToJsonObj(BookTuple obj) => new {
-         secuirty = obj.security.ToString(),
+         security = obj.security.ToString(),
          book = obj.book.ToJsonObj()
       };
       public static bool operator ==(BookTuple left, BookTuple right) =>

src/client/spark_api/MarketData/Co2NativeMethods.cs

@@ -67,7 +67,10 @@ namespace mbts.co2 {
       [DllImport("libspark")]
       internal unsafe static extern int md_consumer_read_trades(
          IntPtr consumer, UserSymbol channel, ref long cursor, raw_receive_pb cb);
-
+      [DllImport("libspark")]
+      internal unsafe static extern int md_consumer_read_trades_linux(
+         IntPtr consumer, string symbol, int venue, int type, ref long cursor, raw_receive_pb cb);
+         
       [DllImport("libspark")]
       internal unsafe static extern int md_consumer_read_rfqs(
          IntPtr consumer, UserSymbol channel, ref long cursor, raw_receive_pb cb);

src/client/spark_api/MarketData/MDConsumerChannel.cs

@@ -23,7 +23,7 @@ namespace mbts.co2 {
 
       public unsafe IList<TradeMessage> read_trades(ref long cursor) {
          IList<TradeMessage> trades = Array.Empty<TradeMessage>();
-         Co2NativeMethods.md_consumer_read_trades(_consumer.Handle, _symbol, ref cursor, callback);
+         Co2NativeMethods.md_consumer_read_trades_linux(_consumer.Handle, _symbol.symbol,(int)_symbol.exchange,(int)_symbol.type , ref cursor, callback);
          return trades;
 
          void callback(byte* buffer, ulong size) =>

src/client/spark_api/MarketData/MarketDataClient.cs

@@ -1,287 +1,280 @@
-#nullable enable
-namespace bts.md {
-   using bts.api;
-   using bts.api.core;
-   using bts.collections;
-
-   using mbts;
-   using mbts.co2;
-   using mbts.spark;
-
-   using Spark.Api.MarketData;
-   using Spark.Core;
-   using Spark.Core.Reactive;
-
-   using System;
-   using System.Collections.Generic;
-   using System.Linq;
-   using System.Reactive.Linq;
-
-   public class MarketDataClient {
-      readonly MDSource _source;
-      readonly TimeSpan _poll_interval;
-      readonly TimeSpan _slow_poll_interval;
-      readonly uint _num_depth_levels;
-
-      public MarketDataClient(MDSource source)
-         : this(source, 5, TimeSpan.FromSeconds(0.1), TimeSpan.FromSeconds(1)) { }
-
-      public MarketDataClient(
-         MDSource source,
-         uint num_depth_levels,
-         TimeSpan poll_interval,
-         TimeSpan slow_poll_interval) {
-         _source = source;
-         _poll_interval = poll_interval;
-         if (slow_poll_interval < poll_interval) {
-            Log.Api.warn("MarketDataClient 'slow_poll_interval' smaller than 'poll_interval'");
-         }
-         _slow_poll_interval = slow_poll_interval;
-         _num_depth_levels = num_depth_levels;
-      }
-
-      public IObservable<IReadOnlyDictionary<uint, SecurityImmutable>> securities(
-         SecurityCriteria? criteria = null) =>
-         criteria.subscribe(_source.subscribe_securities())
-            .Select(secs => secs.ToDictionary(s => s.Id));
-
-      public IObservable<TableUpdate<SecurityImmutable, uint>> securities_incremental(
-         SecurityCriteria? criteria = null) =>
-         criteria is null ? securities_incremental() : securities_incremental(criteria.includes);
-
-      public IObservable<TableUpdate<SecurityImmutable, uint>> securities_incremental(
-         Func<SecurityImmutable, bool> filter) {
-         var secs_src = _source.subscribe_securities().ToSelector(
-            sec_mds => MappedDictionary.create(sec_mds, s => s.security));
-         if (filter is not null) {
-            secs_src = secs_src.extend(
-               secs => new FilteredDictionary<uint, SecurityImmutable>(secs, filter));
-         }
-         return secs_src.Table(s => s.Id);
-      }
-
-      public IObservable<TableUpdate<BookTuple, uint>> books(SecurityCriteria? criteria = null) =>
-         books(criteria is null ? null :
-            new Func<SecurityImmutable, bool>(s => criteria.includes(s)));
-
-      public IObservable<TableUpdate<BookTuple, uint>> books(Func<SecurityImmutable, bool>? filter) {
-         // This MDSource's IObservable provides callbacks when a security is added or removed, but
-         // doesn't detect book changes. To avoid maintaining complicated state, create a brand new
-         // IObservable of book updates whenever the securities change.
-         var secs = _source.subscribe_securities();
-         var filtered_secs = filter is null ?
-            secs.Select(mds => mds.Values) :
-            secs.Select(mds => mds.Values.Where(md => filter(md.security)));
-         return filtered_secs.Select(fixed_sec_market_data).FlattenTableSeries();
-
-         IObservable<TableUpdate<BookTuple, uint>> fixed_sec_market_data(
-            IEnumerable<SecurityMD> security_mds) {
-            // Convert 'security_mds' to a list of PollableBooks.
-            var cached_books = security_mds
-               .Select(s => new PollableBook(s, _num_depth_levels))
-               .ToList();
-
-            return Observable.Interval(_poll_interval).Scan(
-               TableUpdate<BookTuple, uint>.mk_empty(get_sec_id),
-               poll_books);
-
-            uint get_sec_id(BookTuple book) =>
-               book.security.Id;
-
-            TableUpdate<BookTuple, uint> poll_books(TableUpdate<BookTuple, uint> table, long _) {
-               var bd = table.to_builder();
-               foreach (var cbook in cached_books) {
-                  if (cbook.poll_changes()) {
-                     bd.add_or_update(cbook.to_book_tuple());
-                  }
-               }
-               return bd.to_table();
-            }
-         }
-      }
-
-      public IObservable<IList<TradeInfo>> trades(TradesSubscriptionFilter filter) {
-         var sec_mds = filter.security_criteria == null ?
-            _source.subscribe_securities() :
-            filter.security_criteria.subscribe_md_map(_source.subscribe_securities());
-         return TradesSource.create(_source.consumer, sec_mds, _poll_interval);
-      }
-
-      public IObservable<TableUpdate<PriceTicker, uint>> price_ticker(
-         SecurityCriteria? criteria = null) {
-         // This MDSource's IObservable provides callbacks when a security is added or removed, but
-         // doesn't detect ticker changes. To avoid maintaining complicated state, create a brand new
-         // IObservable of ticker updates whenever the securities change.
-         return criteria.subscribe_md(_source.subscribe_securities())
-            .Select(fixed_sec_price_tickers)
-            .FlattenTableSeries();
-
-         IObservable<TableUpdate<PriceTicker, uint>> fixed_sec_price_tickers(
-            IEnumerable<SecurityMD> security_mds) {
-            // Convert 'security_mds' to a list of PollablePriceTickers.
-            var cached_tickers = security_mds
-               .Select(s => new PollablePriceTicker(s))
-               .ToList();
-
-            return Observable.Interval(_poll_interval).Scan(
-               TableUpdate<PriceTicker, uint>.mk_empty(get_sec_id),
-               poll_tickers);
-
-            uint get_sec_id(PriceTicker ticker) =>
-               ticker.security.Id;
-
-            TableUpdate<PriceTicker, uint> poll_tickers(TableUpdate<PriceTicker, uint> table, long _) {
-               var bd = table.to_builder();
-               foreach (var cticker in cached_tickers) {
-                  if (cticker.poll_changes()) {
-                     bd.add_or_update(cticker.to_ticker());
-                  }
-               }
-               return bd.to_table();
-            }
-         }
-      }
-
-      public IObservable<TableUpdate<PriceStats, uint>> price_stats(
-         SecurityCriteria? criteria = null) {
-         // This MDSource's IObservable provides callbacks when a security is added or removed, but
-         // doesn't detect stats changes. To avoid maintaining complicated state, create a brand new
-         // IObservable of stats updates whenever the securities change.
-         return criteria.subscribe_md(_source.subscribe_securities())
-            .Select(fixed_sec_price_stats)
-            .FlattenTableSeries();
-
-         IObservable<TableUpdate<PriceStats, uint>> fixed_sec_price_stats(
-            IEnumerable<SecurityMD> security_mds) {
-            // Convert 'security_mds' to a list of PollablePriceStats.
-            List<PollablePriceStats>? cached_statss = security_mds
-               .Select(s => new PollablePriceStats(s))
-               .ToList();
-            var seed = TableUpdate<PriceStats, uint>.mk_empty(get_sec_id);
-            seed = get_seed(seed);
-            return Observable.Interval(_slow_poll_interval).Scan(
-                  seed,
-               poll_stats);
-
-            uint get_sec_id(PriceStats stats) =>
-               stats.security.Id;
-            TableUpdate<PriceStats, uint> get_seed(TableUpdate<PriceStats, uint> table) {
-               if (cached_statss == null) {
-                  return table;
-               }
-               var bd = table.to_builder();
-               foreach (var cstats in cached_statss) {
-                  bd.add_or_update(cstats.to_stats());
-               }
-               return bd.to_table();
-            }
-            TableUpdate<PriceStats, uint> poll_stats(TableUpdate<PriceStats, uint> table, long _) {
-               var bd = table.to_builder();
-               foreach (var cstats in cached_statss) {
-                  if (cstats.poll_changes()) {
-                     bd.add_or_update(cstats.to_stats());
-                  }
-               }
-               return bd.to_table();
-            }
-         }
-      }
-
-      class PollableBook {
-         readonly SecurityMD _security_md;
-         readonly uint _num_depth_levels;
-         ulong _last_update_time = 0;
-         // Reuse the same arrays as an optimization to avoid reallocating memory on each poll.
-         // Note: this will break if the code isn't called in a thread-safe manner.
-         NativePriceSize[] _bids;
-         NativePriceSize[] _asks;
-
-         public PollableBook(SecurityMD security_md, uint num_depth_levels) {
-            _security_md = security_md;
-            _num_depth_levels = num_depth_levels;
-            _bids = new NativePriceSize[_num_depth_levels];
-            _asks = new NativePriceSize[_num_depth_levels];
-         }
-
-         public BookTuple to_book_tuple() {
-            var book = new Book(
-               _security_md.security.Id, _to_book_side(_bids), _to_book_side(_asks));
-            return new(_security_md.security, book, _security_md.quantity_display);
-         }
-
-         public bool poll_changes() =>
-            _security_md.md.get_depth(
-               ref _last_update_time, ref _bids, ref _asks, _num_depth_levels);
-
-         static List<PriceSize> _to_book_side(NativePriceSize[] native_book_side) =>
-            native_book_side
-               .Select(nps => new PriceSize(nps.size, nps.direct_size, nps.implied_size, nps.price))
-               .ToList();
-      }
-
-      class PollablePriceTicker {
-         readonly SecurityMD _security_md;
-         ulong _bbo_poll_time = 0;
-         ulong _last_poll_time = 0;
-         NativePriceSize _bid;
-         NativePriceSize _ask;
-         NativePriceSize _last;
-         PriceTicker _ticker;
-         ulong _volume;
-
-         public PollablePriceTicker(SecurityMD security_md) =>
-            _security_md = security_md;
-
-         public PriceTicker to_ticker() =>
-            _ticker;
-
-         public bool poll_changes() {
-            var md = _security_md.md;
-            md.get_bbo(ref _bbo_poll_time, out _bid, out _ask);
-            md.get_last_trade(ref _last_poll_time, out _last, out _, out _volume);
-            return Swap.toggle_to(ref _ticker, _mk_ticker());
-         }
-
-         PriceTicker _mk_ticker() {
-            var md = _security_md.md;
-            return new(
-               _security_md.security,
-               _security_md.quantity_display,
-               md.session_high,
-               md.session_low,
-               _volume,
-               _bid.price,
-               _bid.size,
-               _ask.price,
-               _ask.size,
-               md.fixing_price,
-               _last.price,
-               _last.size);
-         }
-      }
-
-      class PollablePriceStats {
-         readonly SecurityMD _security_md;
-         PriceStats _stats;
-
-         public PollablePriceStats(SecurityMD security_md) {
-            _security_md = security_md;
-            _stats = _mk_stats();
-         }
-
-         public PriceStats to_stats() =>
-            _stats;
-
-         public bool poll_changes() =>
-            Swap.toggle_to(ref _stats, _mk_stats());
-         PriceStats _mk_stats() {
-            var md = _security_md.md;
-            return new(
-               _security_md.security,
-               md.open_price,
-               md.close_price,
-               md.settlement_price);
-         }
-      }
-   }
-}
+#nullable enable
+namespace bts.md {
+   using bts.api;
+   using bts.api.core;
+   using bts.collections;
+   using bts.utils;
+
+   using mbts;
+   using mbts.co2;
+   using mbts.spark;
+
+   using Spark.Api.MarketData;
+   using Spark.Core;
+   using Spark.Core.Reactive;
+
+   using System;
+   using System.Collections.Generic;
+   using System.Linq;
+   using System.Reactive.Linq;
+
+   public class MarketDataClient {
+      readonly MDSource _source;
+      readonly TimeSpan _poll_interval;
+      readonly TimeSpan _slow_poll_interval;
+      readonly uint _num_depth_levels;
+
+      public MarketDataClient(MDSource source)
+         : this(source, 5, TimeSpan.FromSeconds(0.1), TimeSpan.FromSeconds(1)) { }
+
+      public MarketDataClient(
+         MDSource source,
+         uint num_depth_levels,
+         TimeSpan poll_interval,
+         TimeSpan slow_poll_interval) {
+         _source = source;
+         _poll_interval = poll_interval;
+         if (slow_poll_interval < poll_interval) {
+            Log.Api.warn("MarketDataClient 'slow_poll_interval' smaller than 'poll_interval'");
+         }
+         _slow_poll_interval = slow_poll_interval;
+         _num_depth_levels = num_depth_levels;
+      }
+
+      public IObservable<IReadOnlyDictionary<uint, SecurityImmutable>> securities(
+         SecurityCriteria? criteria = null) =>
+         criteria.subscribe(_source.subscribe_securities())
+            .Select(secs => secs.ToDictionary(s => s.Id));
+
+      public IObservable<TableUpdate<SecurityImmutable, uint>> securities_incremental(
+         SecurityCriteria? criteria = null) =>
+         criteria is null ? securities_incremental() : securities_incremental(criteria.includes);
+
+      public IObservable<TableUpdate<SecurityImmutable, uint>> securities_incremental(
+         Func<SecurityImmutable, bool> filter) {
+         var secs_src = _source.subscribe_securities().ToSelector(
+            sec_mds => MappedDictionary.create(sec_mds, s => s.security));
+         if (filter is not null) {
+            secs_src = secs_src.extend(
+               secs => new FilteredDictionary<uint, SecurityImmutable>(secs, filter));
+         }
+         return secs_src.Table(s => s.Id);
+      }
+
+      public IObservable<TableUpdate<BookTuple, uint>> books(SecurityCriteria? criteria = null) =>
+         books(criteria is null ? null :
+            new Func<SecurityImmutable, bool>(s => criteria.includes(s)));
+
+      public IObservable<TableUpdate<BookTuple, uint>> books(Func<SecurityImmutable, bool>? filter) {
+         // This MDSource's IObservable provides callbacks when a security is added or removed, but
+         // doesn't detect book changes. To avoid maintaining complicated state, create a brand new
+         // IObservable of book updates whenever the securities change.
+         var secs = _source.subscribe_securities();
+         var filtered_secs = filter is null ?
+            secs.Select(mds => mds.Values) :
+            secs.Select(mds => mds.Values.Where(md => filter(md.security)));
+         return filtered_secs.Select(fixed_sec_market_data).FlattenTableSeries();
+
+         IObservable<TableUpdate<BookTuple, uint>> fixed_sec_market_data(
+            IEnumerable<SecurityMD> security_mds) {
+            // Convert 'security_mds' to a list of PollableBooks.
+            var cached_books = security_mds
+               .Select(s => new PollableBook(s, _num_depth_levels))
+               .ToList();
+
+            return Observable.Interval(_poll_interval).Scan(
+               TableUpdate<BookTuple, uint>.mk_empty(get_sec_id),
+               poll_books);
+
+            uint get_sec_id(BookTuple book) =>
+               book.security.Id;
+
+            TableUpdate<BookTuple, uint> poll_books(TableUpdate<BookTuple, uint> table, long _) {
+               var bd = table.to_builder();
+               foreach (var cbook in cached_books) {
+                  if (cbook.poll_changes()) {
+                     bd.add_or_update(cbook.to_book_tuple());
+                  }
+               }
+               return bd.to_table();
+            }
+         }
+      }
+
+      public IObservable<IList<TradeInfo>> trades(TradesSubscriptionFilter filter) {
+         var sec_mds = filter.security_criteria == null ?
+            _source.subscribe_securities() :
+            filter.security_criteria.subscribe_md_map(_source.subscribe_securities());
+         return TradesSource.create(_source.consumer, sec_mds, _poll_interval);
+      }
+
+      public IObservable<TableUpdate<PriceTicker, uint>> price_ticker(
+         SecurityCriteria? criteria = null) {
+         // This MDSource's IObservable provides callbacks when a security is added or removed, but
+         // doesn't detect ticker changes. To avoid maintaining complicated state, create a brand new
+         // IObservable of ticker updates whenever the securities change.
+         return criteria.subscribe_md(_source.subscribe_securities())
+            .Select(fixed_sec_price_tickers)
+            .FlattenTableSeries();
+
+         IObservable<TableUpdate<PriceTicker, uint>> fixed_sec_price_tickers(
+            IEnumerable<SecurityMD> security_mds) {
+            // Convert 'security_mds' to a list of PollablePriceTickers.
+            var cached_tickers = security_mds
+               .Select(s => new PollablePriceTicker(s))
+               .ToList();
+
+            return Observable.Interval(_poll_interval).Scan(
+               TableUpdate<PriceTicker, uint>.mk_empty(get_sec_id),
+               poll_tickers);
+
+            uint get_sec_id(PriceTicker ticker) =>
+               ticker.security.Id;
+
+            TableUpdate<PriceTicker, uint> poll_tickers(TableUpdate<PriceTicker, uint> table, long _) {
+               bool is_first_update = table.state.Count == 0;
+               var bd = table.to_builder();
+               foreach (var cticker in cached_tickers) {
+                  if (cticker.poll_changes() || is_first_update) {
+                     bd.add_or_update(cticker.to_ticker());
+                  }
+               }
+               return bd.to_table();
+            }
+         }
+      }
+
+      public IObservable<TableUpdate<PriceStats, uint>> price_stats(
+         SecurityCriteria? criteria = null) {
+         // This MDSource's IObservable provides callbacks when a security is added or removed, but
+         // doesn't detect stats changes. To avoid maintaining complicated state, create a brand new
+         // IObservable of stats updates whenever the securities change.
+         return criteria.subscribe_md(_source.subscribe_securities())
+            .Select(fixed_sec_price_stats)
+            .FlattenTableSeries();
+
+         IObservable<TableUpdate<PriceStats, uint>> fixed_sec_price_stats(
+            IEnumerable<SecurityMD> security_mds) {
+            // Convert 'security_mds' to a list of PollablePriceStats.
+            var cached_statss = security_mds
+               .Select(s => new PollablePriceStats(s))
+               .ToList();
+
+            return Observable.Interval(_slow_poll_interval).Scan(
+               TableUpdate<PriceStats, uint>.mk_empty(get_sec_id),
+               poll_stats);
+
+            uint get_sec_id(PriceStats stats) =>
+               stats.security.Id;
+
+            TableUpdate<PriceStats, uint> poll_stats(TableUpdate<PriceStats, uint> table, long _) {
+               bool is_first_update = table.state.Count == 0;
+               var bd = table.to_builder();
+               foreach (var cstats in cached_statss) {
+                  if (cstats.poll_changes() || is_first_update) {
+                     bd.add_or_update(cstats.to_stats());
+                  }
+               }
+               return bd.to_table();
+            }
+         }
+      }
+
+      class PollableBook {
+         readonly SecurityMD _security_md;
+         readonly uint _num_depth_levels;
+         ulong _last_update_time = 0;
+         // Reuse the same arrays as an optimization to avoid reallocating memory on each poll.
+         // Note: this will break if the code isn't called in a thread-safe manner.
+         NativePriceSize[] _bids;
+         NativePriceSize[] _asks;
+
+         public PollableBook(SecurityMD security_md, uint num_depth_levels) {
+            _security_md = security_md;
+            _num_depth_levels = num_depth_levels;
+            _bids = new NativePriceSize[_num_depth_levels];
+            _asks = new NativePriceSize[_num_depth_levels];
+         }
+
+         public BookTuple to_book_tuple() {
+            var book = new Book(
+               _security_md.security.Id, _to_book_side(_bids), _to_book_side(_asks));
+            return new(_security_md.security, book, _security_md.quantity_display);
+         }
+
+         public bool poll_changes() =>
+            _security_md.md.get_depth(
+               ref _last_update_time, ref _bids, ref _asks, _num_depth_levels);
+
+         static List<PriceSize> _to_book_side(NativePriceSize[] native_book_side) =>
+            native_book_side
+               .Select(nps => new PriceSize(nps.size, nps.direct_size, nps.implied_size, nps.price))
+               .ToList();
+      }
+
+      class PollablePriceTicker {
+         readonly SecurityMD _security_md;
+         ulong _bbo_poll_time = 0;
+         ulong _last_poll_time = 0;
+         NativePriceSize _bid;
+         NativePriceSize _ask;
+         NativePriceSize _last;
+         PriceTicker _ticker;
+         ulong _volume;
+
+         public PollablePriceTicker(SecurityMD security_md) =>
+            _security_md = security_md;
+
+         public PriceTicker to_ticker() =>
+            _ticker;
+
+         public bool poll_changes() {
+            var md = _security_md.md;
+            md.get_bbo(ref _bbo_poll_time, out _bid, out _ask);
+            md.get_last_trade(ref _last_poll_time, out _last, out _, out _volume);
+            return Swap.toggle_to(ref _ticker, _mk_ticker());
+         }
+
+         PriceTicker _mk_ticker() {
+            var md = _security_md.md;
+            return new(
+               _security_md.security,
+               _security_md.quantity_display,
+               md.session_high,
+               md.session_low,
+               _volume,
+               _bid.price,
+               _bid.size,
+               _ask.price,
+               _ask.size,
+               md.fixing_price,
+               _last.price,
+               _last.size);
+         }
+      }
+
+      class PollablePriceStats {
+         readonly SecurityMD _security_md;
+         PriceStats _stats;
+
+         public PollablePriceStats(SecurityMD security_md) {
+            _security_md = security_md;
+            _stats = _mk_stats();
+         }
+
+         public PriceStats to_stats() =>
+            _stats;
+
+         public bool poll_changes() =>
+            Swap.toggle_to(ref _stats, _mk_stats());
+         PriceStats _mk_stats() {
+            var md = _security_md.md;
+            return new(
+               _security_md.security,
+               md.open_price,
+               md.close_price,
+               md.settlement_price);
+         }
+      }
+   }
+}

src/client/spark_api/MarketData/TradeInfo.cs

@@ -28,7 +28,7 @@ namespace mbts.spark {
       public TickDisplaySettingsImmutable qty_display => security_md.quantity_display;
 
       public static System.Object ToJsonObj(TradeInfo obj) => new {
-         secuirty = obj.security.Exchange.ToString() + "." + obj.security.ToString(),
+         security = obj.security.Exchange.ToString() + "." + obj.security.ToString(),
          price = obj.trade.price(),
          timestamp = obj.trade.Time,
          quantity = obj.volume,

src/client/spark_api/Orders/OrderDataHelper.cs

@@ -1,5 +1,4 @@
-using bts;
-using bts.api;
+using bts.api;
 using bts.api.core;
 using bts.api.orderman;
 using bts.collections;
@@ -10,23 +9,19 @@ using Google.Protobuf;
 using Google.Protobuf.Reflection;
 using Google.Protobuf.WellKnownTypes;
 
-using mbts.ui;
-using mbts.ui.utilities;
 using mbts.ui.utility;
 using mbts.utils;
 
-using Spark.Core.Disposables;
-
 using System;
 using System.Collections.Generic;
-using System.Composition;
 using System.Linq;
 using System.Reactive.Linq;
+using System.Reflection;
 
 using OrderState = bts.api.orderman.OrderState;
 
 namespace mbts.spark.utilities {
-   struct ChildAlgoData {
+   public readonly struct ChildAlgoData {
       public readonly string name;
       public readonly Func<IMessage, CustomOrderDefinition> get_child_from_impl;
       public readonly Action<IMessage, CustomOrderDefinition> assign_child_to_impl;
@@ -48,27 +43,29 @@ namespace mbts.spark.utilities {
       }
    }
 
-   static class OrderDataHelper {
-      class AlgoTypeCaseDescriptorData {
-         public readonly IReadOnlyList<ChildAlgoData> child_algo_data;
-         public readonly IReadOnlyList<FieldDescriptor> value_descriptors;
-         public readonly FieldDescriptor type_case_descriptor;
+   public class AlgoTypeCaseDescriptorData {
+      public readonly IReadOnlyList<ChildAlgoData> child_algo_data;
+      public readonly IReadOnlyList<FieldDescriptor> value_descriptors;
+      public readonly FieldDescriptor type_case_descriptor;
 
-         public AlgoTypeCaseDescriptorData(FieldDescriptor type_case_descriptor, IReadOnlyList<ChildAlgoData> child_algo_data, IReadOnlyList<FieldDescriptor> value_descriptors) {
-            this.type_case_descriptor = type_case_descriptor;
-            this.child_algo_data = child_algo_data;
-            this.value_descriptors = value_descriptors;
-         }
+      public AlgoTypeCaseDescriptorData(
+         FieldDescriptor type_case_descriptor,
+         IReadOnlyList<ChildAlgoData> child_algo_data,
+         IReadOnlyList<FieldDescriptor> value_descriptors
+      ) {
+         this.type_case_descriptor = type_case_descriptor;
+         this.child_algo_data = child_algo_data;
+         this.value_descriptors = value_descriptors;
       }
+   }
 
+   public static class OrderDataHelper {
       const string _primary_child_field = "child";
       // algo-type-case maps
       // Note: obv this will break if we change the name of the 'CustomOneofCase' field
       static readonly OneofDescriptor _oneof_desc = CustomOrderDefinition.Descriptor.Oneofs.FirstOrDefault(fd => fd.Name == "custom");
       static readonly IReadOnlyDictionary<System.Type, CustomOrderDefinition.CustomOneofCase> _algo_type_case_index;
       static readonly IReadOnlyDictionary<CustomOrderDefinition.CustomOneofCase, AlgoTypeCaseDescriptorData> _algo_type_case_descriptor_data;
-      // 
-      static readonly OrderMetadata _default_order_metadata = new OrderMetadata { KeyBinding = "", Color = BTSColors.Orders.default_order };
 
       static OrderDataHelper() {
          var algo_type_case_index = new Dictionary<System.Type, CustomOrderDefinition.CustomOneofCase>();
@@ -106,6 +103,12 @@ namespace mbts.spark.utilities {
       }
 
       #region Algo Structure/Hierarchy
+      public static AlgoTypeCaseDescriptorData get_algo_type_descriptor(
+         CustomOrderDefinition.CustomOneofCase algo_type) =>
+         _algo_type_case_descriptor_data.TryGetValue(algo_type, out var d) ? d :
+            throw new ArgumentException(
+               $"Failed to find descriptor data for algo type '{algo_type}'", nameof(algo_type));
+
       public static IEnumerable<(CustomOrderDefinition.CustomOneofCase type, string dyn, uint studio, uint spreader_sec)> enumerate_algo_structure(CustomOrderDefinitionImmutable custom_defi) {
          string dyn = null;
          uint studio = 0, spreader_sec = 0;
@@ -205,29 +208,6 @@ namespace mbts.spark.utilities {
          _algo_type_case_descriptor_data.TryGetValue(algo_type, out AlgoTypeCaseDescriptorData d) ? d.child_algo_data : Array.Empty<ChildAlgoData>();
       #endregion
 
-      public static OrderMetadata get_metadata(CustomOrderDefinition.CustomOneofCase order_type) =>
-         order_type switch {
-            CustomOrderDefinition.CustomOneofCase.Market => new() { KeyBinding = "M", Color = BTSColors.Orders.market_order, NoProps = true },
-            CustomOrderDefinition.CustomOneofCase.Limit => new() { KeyBinding = "L", Color = BTSColors.Orders.limit_order, NoProps = true },
-            CustomOrderDefinition.CustomOneofCase.Stop => new() { KeyBinding = "S", Color = BTSColors.Orders.stop_order, NoProps = true },
-            CustomOrderDefinition.CustomOneofCase.StopLimit => new() { KeyBinding = "SL", Color = BTSColors.Orders.stop_limit_order },
-            CustomOrderDefinition.CustomOneofCase.GlbxIceberg => new() { KeyBinding = "GB", Color = BTSColors.Orders.glbx_iceberg_order },
-            CustomOrderDefinition.CustomOneofCase.Iceberg => new() { KeyBinding = "I", Color = BTSColors.Orders.iceberg_order },
-            CustomOrderDefinition.CustomOneofCase.TimeBox => new() { KeyBinding = "TB", Color = BTSColors.Orders.time_box_order },
-            CustomOrderDefinition.CustomOneofCase.Tick => new() { KeyBinding = "T", Color = BTSColors.Orders.tick_order },
-            CustomOrderDefinition.CustomOneofCase.Lean => new() { KeyBinding = "L", Color = BTSColors.Orders.lean_order },
-            CustomOrderDefinition.CustomOneofCase.Join => new() { KeyBinding = "J", Color = BTSColors.Orders.join_order },
-            CustomOrderDefinition.CustomOneofCase.Top => new() { KeyBinding = "D", Color = BTSColors.Orders.top_order },
-            CustomOrderDefinition.CustomOneofCase.Cover => new() { KeyBinding = "C", Color = BTSColors.Orders.cover_order },
-            CustomOrderDefinition.CustomOneofCase.Oco => new() { KeyBinding = "C", Color = BTSColors.Orders.oco_order },
-            CustomOrderDefinition.CustomOneofCase.Pricer => new() { KeyBinding = "C", Color = BTSColors.Orders.pricer_order },
-            CustomOrderDefinition.CustomOneofCase.Offset => new() { KeyBinding = "O", Color = BTSColors.Orders.off_order },
-            CustomOrderDefinition.CustomOneofCase.Trigger => new() { KeyBinding = "T", Color = BTSColors.Orders.trigger_order },
-            CustomOrderDefinition.CustomOneofCase.Spreader => new() { KeyBinding = "", Color = BTSColors.Orders.spreader_order },
-            CustomOrderDefinition.CustomOneofCase.Dynamic => new() { KeyBinding = "", Color = BTSColors.Orders.dynamic_order },
-            _ => _default_order_metadata,
-         };
-
       public static string abbreviate_algo_type(CustomOrderDefinition.CustomOneofCase algo_type) {
          switch (algo_type) {
             case CustomOrderDefinition.CustomOneofCase.Limit: return "LM";
@@ -331,6 +311,19 @@ namespace mbts.spark.utilities {
          order_type_def != null && _algo_type_case_index.TryGetValue(order_type_def.Descriptor.ClrType, out CustomOrderDefinition.CustomOneofCase algo_case) ?
             algo_case : CustomOrderDefinition.CustomOneofCase.None;
 
+      public static void setup_generic_custom_order(
+         CustomOrderDefinition custom, CustomOrderDefinition.CustomOneofCase type) {
+         var prop = typeof(CustomOrderDefinition).GetProperty(type.ToString(),
+            BindingFlags.Instance | BindingFlags.Public);
+         var rt = prop.PropertyType;
+         var rtt = Activator.CreateInstance(rt);
+
+         var ci = rt.GetProperties().Where(x => x.PropertyType == typeof(CustomOrderDefinition));
+         foreach (var cid in ci)
+            cid.SetValue(rtt, new CustomOrderDefinition { Limit = new LimitOrderDefinition() });
+         prop.SetValue(custom, rtt);
+      }
+
       #region Private Members
       // get child
       static CustomOrderDefinitionImmutable? _get_iceberg_child(IcebergOrderDefinitionImmutable iceberg) => iceberg.HasChild ? iceberg.Child : (CustomOrderDefinitionImmutable?)null;
@@ -655,7 +648,7 @@ namespace mbts.spark.utilities {
                case bts.api.orderman.CustomOrderDefinition.CustomOneofCase.Spreader: rv.Spreader = SpreaderOrderDefinition(); break;
                case bts.api.orderman.CustomOrderDefinition.CustomOneofCase.Dynamic: rv.Dynamic = DynamicOrderDefinition(); break;
                default:
-                  OrderTypeVMFactory.GenericOrderFactory.setup_generic_custom_order(rv, order_type);
+                  setup_generic_custom_order(rv, order_type);
                   break;
             }
             return rv;

src/client/spark_api/Orders/OrdersClient.cs

@@ -1,6 +1,8 @@
 #nullable enable
 namespace Spark.Api.Orders {
    using bts;
+   using bts.api;
+
    using bts.api.core;
    using bts.api.orderman;
    using bts.api.position;
@@ -14,6 +16,8 @@ namespace Spark.Api.Orders {
    using Spark.Core.Reactive;
 
    using System;
+   using System.Collections.Generic;
+
    using System.Reactive.Concurrency;
    using System.Reactive.Linq;
    using System.Threading.Tasks;
@@ -55,7 +59,7 @@ namespace Spark.Api.Orders {
                o => o.order.ParentKey,
                (o, r) => new OrderRecordTupleWithRoot(o, r),
                o => new OrderRecordTupleWithRoot(o),
-               o => o.root.order.Key)
+               o => o.key)
                .Cache()
                .ToObservable());
 
@@ -127,11 +131,29 @@ namespace Spark.Api.Orders {
             });
             return r.value;
          } catch {
-            var r = new Response();
-            r.ErrorCode = OrderError.Undefined;
+            var r = new Response {
+               ErrorCode = OrderError.Undefined
+            };
             return r;
          }
       }
+      public async Task<Response> TrySetPrecomputedTheosAsync(
+         List<PrecomputedTheoPoint> ptp_list
+      ) {
+         bts.api.orderman.PrecomputedTheos server_theos = new();
+         server_theos.Theos.AddRange(ptp_list);
+         var r = await _order_service.set_precomputed_theos_async(server_theos);
+         var r2 = new Response {
+            ErrorCode = OrderError.Ok,
+            Msg = r.ToString()
+         };
+         return r2;
+      }
+
+      public async Task<RegisteredDynamicOrderTypes> GetRegisteredDynamicOrderTypesAsync(){
+         Result<RegisteredDynamicOrderTypes> r = await _order_service.get_registered_dynamic_order_types_async(new bts.api.VoidMessage());
+         return r.value;
+      }
       public async Task<Response> TryUpdateOrderAsync(
          CustomOrderDefinition custom,
          double price, double quantity, uint order_id
@@ -146,8 +168,9 @@ namespace Spark.Api.Orders {
             });
             return r.value;
          } catch {
-            var r = new Response();
-            r.ErrorCode = OrderError.Undefined;
+            var r = new Response {
+               ErrorCode = OrderError.Undefined
+            };
             return r;
          }
       }
@@ -159,8 +182,9 @@ namespace Spark.Api.Orders {
             });
             return r.value;
          } catch {
-            var r = new Response();
-            r.ErrorCode = OrderError.Undefined;
+            var r = new Response {
+               ErrorCode = OrderError.Undefined
+            };
             return r;
          }
       }

src/client/spark_api_server/Program.cs

@@ -44,7 +44,7 @@
             KeepAliveIntervalSecs = 0.1,
          };
          gwy.init(sync);
-         sync.init(gwy);
+         await sync.initAsync(gwy);
          var errors = Errors.None;
          Log.Api.info("Starting websocket server...");
          // try {

src/client/spark_websocket/Core/WebsocketDataSync.cs

@@ -151,6 +151,15 @@ namespace bts.websocket_core {
             _orders_client = value;
          }
       }
+      
+      IReadOnlyDictionary<string, DynamicOrderDescriptor>? _dynamic_descriptors;
+      IReadOnlyDictionary<string, DynamicOrderDescriptor> Dynamic_Descriptors{
+         get => Log.notnull<IReadOnlyDictionary<string, DynamicOrderDescriptor>>(_dynamic_descriptors);
+         set {
+            Log.isnull<IReadOnlyDictionary<string, DynamicOrderDescriptor>>(_dynamic_descriptors);
+            _dynamic_descriptors=value.ToDictionary(k=>k.Key.ToLower(),v => v.Value);
+         }
+      }
       WebsocketConnectionGateway? gateway;
       WebsocketConnectionGateway Gateway {
          get => Log.notnull<WebsocketConnectionGateway>(gateway);
@@ -178,10 +187,10 @@ namespace bts.websocket_core {
       /// Should be called in the start, stores pointer for gtwy and creates dictionary for clients, starts broadcast thread
       /// </summary>
       /// <param name="gtwy"></param>
-      public void init(WebsocketConnectionGateway gtwy) {
+      public async Task initAsync(WebsocketConnectionGateway gtwy) {
          Gateway = gtwy;
          clients = new ClientsStore();
-         TimeSpan md_throttle = new TimeSpan(10000);
+         TimeSpan md_throttle = TimeSpan.FromSeconds(0.1);
          MD_Source = new MDSource(md_throttle);
          Md_Client = new(MD_Source);
          _securities_sub = MD_Source.subscribe_securities().Subscribe(sec_map => _securities = sec_map);
@@ -197,6 +206,10 @@ namespace bts.websocket_core {
          var sec_table_src = MD_Source.subscribe_securities()
             .Table(s => s.security.Id)
             .Select(s => s.security, s => s.Id);
+         var r  = await Orders_Client.GetRegisteredDynamicOrderTypesAsync();
+         Dynamic_Descriptors = r
+         .DynamicOrderDesc
+         .ToDictionary(d => d.DynamicOrderType);
          Synth_Sec_Client = new(synth_sec_service, sec_table_src);
 
 
@@ -270,6 +283,10 @@ namespace bts.websocket_core {
                   handle_update_order(req);
                   break;
                }
+            case WebsocketRequest.REQUEST_TYPES.SET_PRECOMPUTED_THEOS: {
+                  handle_set_theos(req);
+                  break;
+               }
             case WebsocketRequest.REQUEST_TYPES.UNKOWN: {
                   ///send error
                   break;
@@ -284,30 +301,21 @@ namespace bts.websocket_core {
             System.Console.WriteLine("error for req", client_id, ex.ToString());
          }
       }
-      public async void handle_delete_order(WebsocketRequest req) {
-         (bool suc, string status) = await WebsocketRestHandler.DeleteOrderAsync(req, Orders_Client);
-         if (!suc) {
+      public async Task HandleRestRequest(WebsocketRequest req, Func<WebsocketRequest, OrdersClient, Task<(bool success, uint? id, string status)>> handlerAsync) {
+         (bool success, uint? id, string status) = await handlerAsync(req, Orders_Client);
+         if (!success){
             handle_incorrect_request(req, status);
             return;
          }
-         handle_correct_request(req.id, status, null, req.client_id);
-      }
-      public async void handle_enter_order(WebsocketRequest req) {
-         (bool suc, uint? id, string status) = await WebsocketRestHandler.EnterOrderAsync(req, Orders_Client, _securities);
-         if (!suc) {
-            handle_incorrect_request(req, status);
-            return;
-         }
-         handle_correct_request(req.id, status, new { order_key = id }, req.client_id);
-      }
-      public async void handle_update_order(WebsocketRequest req) {
-         (bool suc, string status) = await WebsocketRestHandler.UpdateOrderAsync(req, Orders_Client);
-         if (!suc) {
-            handle_incorrect_request(req, status);
-            return;
-         }
-         handle_correct_request(req.id, status, null, req.client_id);
+         handle_correct_request(req.id, status, id != null ? new { order_key = id } : null, req.client_id);
       }
+      public async void handle_delete_order(WebsocketRequest req) { await HandleRestRequest(req,WebsocketRestHandler.DeleteOrderAsync);} 
+      public async void handle_set_theos(WebsocketRequest req) { await HandleRestRequest(req,(req, Orders_Client) => WebsocketRestHandler.SetTheosAsync(req, Orders_Client, _securities));}
+      public async void handle_enter_order(WebsocketRequest req) { await HandleRestRequest(req,(req, Orders_Client) => WebsocketRestHandler.EnterOrderAsync(
+         req, Orders_Client, _securities,Dynamic_Descriptors  
+      ));} 
+      public async void handle_update_order(WebsocketRequest req) { await HandleRestRequest(req,(req, Orders_Client) => WebsocketRestHandler.UpdateOrderAsync(req,Orders_Client,Dynamic_Descriptors));}
+      
       public void handle_subscription(WebsocketRequest req) {
          WebsocketSubscriptionHandler? hdlr;
          string status;

src/client/spark_websocket/Core/WebsocketJsonParsers.cs

@@ -1,6 +1,10 @@
 namespace bts.websocket_core {
    using bts.api;
    using bts.api.core;
+   using bts.api.orderman;
+
+   using bts.md;
+
 
    using Newtonsoft.Json;
    using Newtonsoft.Json.Linq;
@@ -27,9 +31,67 @@ namespace bts.websocket_core {
       public static Object to_json<T>(IList<T> list)
           where T : IJsonFormatter<T> =>
           to_json(list, T.ToJsonObj);
-      public struct ListWrapper : IEnumerable<string> {
+      public struct PrecomputedTheoPointWrapper {
+         UInt32 SecId{get;init;}
+         public double driver{get;init;}
+         UInt32 DriverSecId{get;init;}
+         public double theo{get;init;}
+         public double delta{get;init;}
+         public double gamma{get;init;}
+         public double vega{get;init;}
+         public double theta{get;init;}
+         public double rho{get;init;}
+         public double vol{get;init;}
+         public Venue venue{get;init;}
+         public String exchange_security_name{get;init;}
+         public String driver_exchange_security_name{get;init;}
+         public static bool GetTheoPoint(PrecomputedTheoPointWrapper inp, [NotNullWhen(true)] out PrecomputedTheoPoint? p, IReadOnlyDictionary<uint, mbts.SecurityMD> _securites){
+            p = null;
+            if (!_securites.Values.TryGetFirst(sc_md =>
+            (
+               sc_md.security.Exchange == inp.venue &&
+               sc_md.security.ToString() == inp.exchange_security_name),
+            out mbts.SecurityMD sec
+            )) {
+            return false;
+            }
+            if (!_securites.Values.TryGetFirst(sc_md =>
+            (
+               sc_md.security.Exchange == inp.venue &&
+               sc_md.security.ToString() == inp.driver_exchange_security_name),
+            out mbts.SecurityMD driver_sec
+            )) {
+            return false;
+            }
+            p = new(){
+               SecId = sec.id,
+               Driver=inp.driver,
+               DriverSecId=driver_sec.id,
+               Theo=inp.theo,
+               Delta=inp.delta,
+               Gamma=inp.gamma,
+               Vega=inp.vega,
+               Theta=inp.theta,
+               Rho=inp.rho,
+               Vol=inp.vol,
+            };
+            return true;
+
+         }
+         public static bool TryParse(string input, [NotNullWhen(true)] out PrecomputedTheoPointWrapper? P) {
+            P = null;
+            try {
+               var temp = JsonConvert.DeserializeObject<PrecomputedTheoPointWrapper>(input);
+               P = temp;
+               return true;
+            } catch {
+               return false;
+            }
+         }
+      }
+      public struct StringListWrapper : IEnumerable<string> {
          public List<string> Lis { get; set; }
-         public ListWrapper(List<string> l) {
+         public StringListWrapper(List<string> l) {
             Lis = l;
          }
 
@@ -49,14 +111,24 @@ namespace bts.websocket_core {
          public string[] getlwrarr() {
             return Lis.Select(v => v.ToLower()).ToArray();
          }
-         public static bool TryParse(string input, [NotNullWhen(true)] out ListWrapper? l) {
+         public static bool TryParse(string input, [NotNullWhen(true)] out StringListWrapper? l) {
             l = null;
             try {
-               var temp = JsonConvert.DeserializeObject<List<string>>(input);
+               var temp = JsonConvert.DeserializeObject<List<Object>>(input);
                if (temp == null) {
                   return false;
                }
-               l = new ListWrapper(temp);
+                List<string> stringList = new List<string>();
+
+               foreach (object obj in temp)
+               {
+                  string? val = obj.ToString();
+                  if(val==null){
+                     return false;
+                  }
+                  stringList.Add(val);
+               }
+               l = new StringListWrapper(stringList);
                return true;
             } catch {
                return false;
@@ -71,26 +143,33 @@ namespace bts.websocket_core {
 
       }
       public static bool TryParse<T>(JObject filter, string key, out T[]? list, out string error) where T : struct {
-         T[] values = new T[] { };
          list = null;
-         ListWrapper? wrap;
-         if (TryParse<ListWrapper>(filter, key, out wrap, out error)) {
+         if (TryParse<StringListWrapper>(filter, key, out StringListWrapper? wrap, out error)) {
             if (wrap != null) {
+                var values = new List<T>();
                foreach (var item in wrap) {
-                  T? obj;
-                  if (TryParse<T>(item, key, out obj, out error) && obj != null) {
-                     values = values.ToList().Append<T>((T)obj).ToArray();
+                  if (TryParse<T>(item, key, out T? obj, out error) && obj != null) {
+                     values.Add(obj.Value);
                   } else {
                      return false;
                   }
                }
-               list = values;
+               list = values.ToArray();
             }
             return true;
          }
          return false;
       }
+      public static bool TryParse<T>(JObject filter, string key, out T? obj, [NotNullWhen(false)] out string error) where T : struct {
+         obj = null;
+         var value = filter.GetValueOrDefault(key)?.ToString();
+         if (value == null) {
+            error = $"{key} not present";
+            return true;
+         }
+         return TryParse<T>(value, key, out obj, out error);
 
+      }
       public static bool TryParse<T>(string value, string key, [NotNullWhen(true)] out T? obj, [NotNull] out string error) where T : struct {
          error = "";
          if (typeof(T).IsEnum && Enum.TryParse<T>(value, true, out var result)) {
@@ -100,7 +179,10 @@ namespace bts.websocket_core {
          if (typeof(T) == typeof(double) && double.TryParse(value, out var d)) {
             obj = (T)(object)d;
             return true;
-
+         }
+         if (typeof(T) == typeof(bool) && bool.TryParse(value, out var b)) {
+            obj = (T)(object)b;
+            return true;
          }
          if (typeof(T) == typeof(uint) && uint.TryParse(value, out var uin)) {
             obj = (T)(object)uin;
@@ -118,10 +200,14 @@ namespace bts.websocket_core {
             obj = (T)(object)new StringWrapper(value);
             return true;
          }
-         if (typeof(T) == typeof(ListWrapper) && ListWrapper.TryParse(value, out var l)) {
+         if (typeof(T) == typeof(StringListWrapper) && StringListWrapper.TryParse(value, out var l)) {
             obj = (T)(object)l;
             return true;
          }
+         if (typeof(T)==typeof(PrecomputedTheoPointWrapper) && PrecomputedTheoPointWrapper.TryParse(value, out var p)){
+            obj=(T)(object)p;
+            return true;
+         }
          if (typeof(T) == typeof(DateTime) && DateTime.TryParseExact(value, "yyyy/MM/dd", CultureInfo.InvariantCulture,
                             DateTimeStyles.None, out var dt)) {
             obj = (T)(object)dt;
@@ -132,16 +218,7 @@ namespace bts.websocket_core {
          obj = null;
          return false;
       }
-      public static bool TryParse<T>(JObject filter, string key, out T? obj, [NotNullWhen(false)] out string error) where T : struct {
-         obj = null;
-         var value = filter.GetValueOrDefault(key)?.ToString();
-         if (value == null) {
-            error = $"{key} not present";
-            return true;
-         }
-         return TryParse<T>(value, key, out obj, out error);
-
-      }
+      
 
    }
 }

src/client/spark_websocket/Core/WebsocketRequest.cs

@@ -12,8 +12,8 @@ namespace bts.websocket_core {
          UNSUBSCRIBE,
          ENTER_ORDER,
          DELETE_ORDER,
-
          UPDATE_ORDER,
+         SET_PRECOMPUTED_THEOS,
          DELETE,
          PATCH,
          UNKOWN = 0

src/client/spark_websocket/Core/WebsocketRestHandler.cs

@@ -6,109 +6,352 @@ namespace bts.websocket_core {
 
    using mbts;
 
+   using Microsoft.VisualBasic;
+
+   using mbts.spark;
    using Newtonsoft.Json.Linq;
 
    using Spark.Api.Orders;
 
    using System;
+   using System.Diagnostics.CodeAnalysis;
+   using bts.spark;
+   using bts.utils;
 
    public class WebsocketRestHandler {
+   public static (bool,DynamicOrderParam?,string) TryGetDynamicParam(JObject custom,string key, DynamicOrderParamDescriptor descriptor, IReadOnlyDictionary<string, DynamicOrderDescriptor> Dynamic_Descriptors) {
+         var rv = new DynamicOrderParam();
+         string error = "";
+         switch (descriptor.ParamType.DescriptorCase) {
+            case DynamicOrderTypeDescriptor.DescriptorOneofCase.Array:
+               if (custom[key] is not JArray arr_obj) {
+                  return (false, null, key + " not correct");
+               }
+               rv.AsArray = new DynamicOrderArray();
+               foreach (var p in arr_obj.ToList()){
+                  var child_key = key+"_child";
+                  DynamicOrderParamDescriptor child_descriptor=new() {
+                     ParamName = child_key,
+                     ParamType = descriptor.ParamType.Array.ElementType
+                  };
+
+                  JObject child = new() {
+                     [child_key] = p
+                  };
+
+                  (bool suc, DynamicOrderParam? param, error) = TryGetDynamicParam(child,child_key,child_descriptor,Dynamic_Descriptors);
+                  if(!suc){
+                     return (suc,null,error);
+                  }
+                  rv.AsArray.Element.Add(param);
+               }
+               return (true,rv,error);
+            case DynamicOrderTypeDescriptor.DescriptorOneofCase.Enumerated:
+               return (false,null,key+"Enumerated DynamicOrderTypeDescriptors not enabled on api");
+            case DynamicOrderTypeDescriptor.DescriptorOneofCase.Primitive:
+               switch (descriptor.ParamType.Primitive) {
+                  case DynamicOrderPrimitiveDescriptor.DynamicOrderPrimitiveBool:
+                     if (!WebsocketJsonParsers.TryParse<bool>(custom, key, out bool? b, out error) || b == null) {
+                        return (false, null, error);
+                     }
+                     rv.AsBool = (bool)b; 
+                     return (true,rv,error);
+                  case DynamicOrderPrimitiveDescriptor.DynamicOrderPrimitiveDouble:
+                     if (!WebsocketJsonParsers.TryParse<double>(custom, key, out double? d, out error) || d == null) {
+                        return (false, null, error);
+                     }
+                     rv.AsDouble = (double)d; 
+                     return (true,rv,error);
+                  case DynamicOrderPrimitiveDescriptor.DynamicOrderPrimitiveInt: 
+                  if (!WebsocketJsonParsers.TryParse<int>(custom, key, out int? i, out error) || i == null) {
+                        return (false, null, error);
+                     }
+                     rv.AsInt32 = (int)i; 
+                     return (true,rv,error);
+                  case DynamicOrderPrimitiveDescriptor.DynamicOrderPrimitiveOrderDef: 
+                     if (custom[key] is not JObject child_custom) {
+                        return (false, null, key + " not correct");
+                     }
+                     (bool suc,var child_custom_def,error) = TryGetCustomDef(child_custom,Dynamic_Descriptors);
+                  if(!suc || child_custom_def==null){
+                     return (false,null,error);
+                  }
+                  rv.AsOrderDef = child_custom_def;
+                  return(true,rv,error);
+                  case DynamicOrderPrimitiveDescriptor.DynamicOrderPrimitiveString: 
+                  if (!WebsocketJsonParsers.TryParse<WebsocketJsonParsers.StringWrapper>(custom,key,out WebsocketJsonParsers.StringWrapper? s,out error)|| s == null) {
+                     return (false, null, error);
+                  }
+                  rv.AsString = s.Value.Val; 
+                  return (true,rv,error);
+                  case DynamicOrderPrimitiveDescriptor.DynamicOrderPrimitiveSecurityKey:
+                  case DynamicOrderPrimitiveDescriptor.DynamicOrderPrimitiveUint: 
+                  if (!WebsocketJsonParsers.TryParse<uint>(custom, key, out uint? u, out error) || u == null) {
+                        return (false, null, error);
+                     }
+                     rv.AsUint32 = (uint)u; 
+                     return (true,rv,error);
+                  case DynamicOrderPrimitiveDescriptor.DynamicOrderPrimitiveQuantity:
+                  if (WebsocketJsonParsers.TryParse<uint>(custom, key, out uint? q, out error) && q != null) {
+                        rv.AsQuantity = new QuantityMeasure() {
+                           Absolute = (uint)q
+                        };
+                        return (true, rv, error);
+                  } else if (WebsocketJsonParsers.TryParse<double>(custom, key, out double? qd, out error) && qd != null){
+                     rv.AsQuantity = new QuantityMeasure() {
+                           Ratio = (double)qd
+                        };
+                        return (true, rv, error);
+                  }
+                  else{
+                     return (false,null,key+" not correct");
+                  }
+               }
+               return (false,null,key+" not correct");
+            case DynamicOrderTypeDescriptor.DescriptorOneofCase.Structure:
+                if (custom[key] is not JObject obj) {
+                  return (false, null, key + " not correct");
+               }
+               rv.AsStruct = new DynamicOrderStruct();
+               
+               foreach (var prop in obj.Properties()) {
+                  var child_key = prop.Name;
+                  var check = descriptor.ParamType.Structure.Field.TryGetFirst(v => v?.ParamName == child_key,out DynamicOrderParamDescriptor? p);
+                  if(!check || p == null){
+                     return (false,null,key+" "+child_key+" not correct");
+                  }
+                  var child_descriptor = new DynamicOrderParamDescriptor {
+                        ParamName = child_key,
+                        ParamType = p.ParamType
+                  };
+
+                  (bool success, DynamicOrderParam? param, error) = TryGetDynamicParam(obj, child_key, child_descriptor, Dynamic_Descriptors);
+                  if (!success) {
+                        return (success, null, error);
+                  }
+                  rv.AsStruct.Field[child_key] = param;
+               }
+               return (true,rv,error);
+         }
+         return (false,null,key+" not correct");
+      }
       readonly static Lazy<AdminService> _admin = new Lazy<AdminService>(() => Locator.locate<AdminService>(), true);
-      private static (bool, CustomOrderDefinition?, string) TryGetCustomDef(WebsocketRequest req) {
-         string status = "";
+      
+      private static (bool, CustomOrderDefinition?, string) TryGetCustomDef(WebsocketRequest req, IReadOnlyDictionary<string, DynamicOrderDescriptor> Dynamic_Descriptors){
+         if (req.parameters["options"] is not JObject custom) {
+            return (false, null, "options not correct");
+         }
+         return TryGetCustomDef(custom,Dynamic_Descriptors);
+      }
+      private static (bool, CustomOrderDefinition?, string) TryGetCustomDef(JObject custom, IReadOnlyDictionary<string, DynamicOrderDescriptor> Dynamic_Descriptors) {
          CustomOrderDefinition customdef;
-         JObject? custom = req.parameters["custom"] as JObject;
-         if (!WebsocketJsonParsers.TryParse<CustomOrderDefinition.CustomOneofCase>(
-            req.parameters,
-            "order_type",
-            out CustomOrderDefinition.CustomOneofCase? order_type,
-            out status)
-            || order_type == null) {
+
+         if (!WebsocketJsonParsers.TryParse<CustomOrderDefinition.CustomOneofCase>(custom, "order_type", out CustomOrderDefinition.CustomOneofCase? order_type, out string status) || order_type == null) {
             return (false, null, status);
          }
          switch (order_type) {
+            case CustomOrderDefinition.CustomOneofCase.Stop:{
+               customdef = new CustomOrderDefinition() {
+                     Stop = new StopOrderDefinition()
+                  };
+                  break;
+            }
+            case CustomOrderDefinition.CustomOneofCase.StopLimit:{
+               if (!WebsocketJsonParsers.TryParse<double>(custom, "stop_price", out double? sp, out status) || sp == null) {
+                     return (false, null, status);
+                  }
+               customdef = new CustomOrderDefinition() {
+                     StopLimit = new StopLimitOrderDefinition(){
+                        StopPrice=(double)sp
+                     }
+                  };
+                  break;
+            }
+            case CustomOrderDefinition.CustomOneofCase.Iceberg: {
+                  if (!WebsocketJsonParsers.TryParse<double>(custom, "show_quantity", out double? sq, out status) || sq == null) {
+                     return (false, null, status);
+                  }
+                  if (custom["child"] is not JObject child) {
+                     return (false, null, "Iceberg child not correct");
+                  }
+                  (var check, var childdef, status) = TryGetCustomDef(child,Dynamic_Descriptors);
+                  if(!check){
+                     return (false, null, status);
+                  }
+                  // mbts.spark.utilities.OrderDataHelper 
+                  customdef = new CustomOrderDefinition() {
+                     Iceberg = new IcebergOrderDefinition() {
+                        ShowQuantity = (double)sq,
+                        Child = childdef
+                     }
+                  };
+                  break;
+               }
+            case CustomOrderDefinition.CustomOneofCase.TimeBox:{
+               return (false,null,"timebox not enabled on api");
+            }
+            case CustomOrderDefinition.CustomOneofCase.Spreader: {
+                  // if (custom == null) {
+                  //    message = "custom not present";
+                  //    return false;
+                  // }
+                  return (false,null,"spreaders not enabled on api");
+               }
+            case CustomOrderDefinition.CustomOneofCase.Tick:{
+               return (false,null,"Ticks not enabled on api");
+            }
+            case CustomOrderDefinition.CustomOneofCase.Lean:{
+               return (false,null,"Lean not enabled on api");
+               // if (!WebsocketJsonParsers.TryParse<double>(custom, "", out double? sq, out status) || sq == null) {
+               //       return (false, null, status);
+               //    }
+               // if (custom["child"] is not JObject child) {
+               //       return (false, null, "Lean child not correct");
+               //    }
+               //    (var check, var childdef, status) = TryGetCustomDef(child,Dynamic_Descriptors);
+               //    if(!check){
+               //       return (false, null, status);
+               //    }
+               
+               // customdef = new CustomOrderDefinition() {
+               //       Lean = new LeanOrderDefinition(){
+               //          DetachOnToward = false,
+               //          CrossThreshold = 0,
+               //          MaxChaseTicks = 0,
+               //          Child = childdef
+
+               //       }
+               //    };
+
+            }
+            case CustomOrderDefinition.CustomOneofCase.Join:{
+               return (false,null,"join not enabled on api");
+            }
+            case CustomOrderDefinition.CustomOneofCase.Top:{
+               return (false,null,"top not enabled on api");
+            }
+            case CustomOrderDefinition.CustomOneofCase.Trigger:{
+               return (false,null,"trigger not enabled on api");
+            }
+            case CustomOrderDefinition.CustomOneofCase.Cover: {
+               if (
+                  !WebsocketJsonParsers.TryParse<uint>( custom, "hedge_ticks", out uint? ht, out status) || ht == null || 
+                  !WebsocketJsonParsers.TryParse<bool>(custom, "cancel_on_external_hedge_cancel", out bool? coehc, out status) || coehc == null ||
+                  !WebsocketJsonParsers.TryParse<bool>(custom, "cancel_hedges_on_cancel", out bool? choc, out status) || choc == null ||
+                  !WebsocketJsonParsers.TryParse<bool>(custom, "detach_hedges", out bool? dh, out status) || dh == null
+                  ) {
+                  return (false, null, status);
+               }
+               if (custom["child"] is not JObject child) {
+                  return (false, null, "Cover child not correct");
+               }
+               (var check, var childdef, status) = TryGetCustomDef(child,Dynamic_Descriptors);
+               if(!check){
+                  return (false, null, status);
+               }
+               if (custom["hedge"] is not JObject hedge) {
+                  return (false, null, "Cover hedge not correct");
+               }
+               (check, var hedgedef, status) = TryGetCustomDef(hedge,Dynamic_Descriptors);
+               if(!check){
+                  return (false, null, status);
+               }
+               customdef = new CustomOrderDefinition() {
+                  Cover = new CoverOrderDefinition() {
+                     HedgeTicks = (uint)ht,
+                     CancelOnExternalHedgeCancel = (bool)coehc,
+                     CancelHedgesOnCancel = (bool)choc,
+                     DetachHedges = (bool)dh,
+                     Hedge = hedgedef,
+                     Child = childdef
+                  }
+               };
+               break;
+            }
+            case CustomOrderDefinition.CustomOneofCase.Offset:{
+               if (!WebsocketJsonParsers.TryParse<uint>(custom, "tick_offset", out uint? to, out status) || to == null) {
+                     return (false, null, status);
+                  }
+                  if (custom["child"] is not JObject child) {
+                     return (false, null, "offset child not correct");
+                  }
+                  (var check, var childdef, status) = TryGetCustomDef(child,Dynamic_Descriptors);
+                  if(!check){
+                     return (false, null, status);
+                  }
+                  // mbts.spark.utilities.OrderDataHelper 
+                  customdef = new CustomOrderDefinition() {
+                     Offset = new OffsetOrderDefinition() {
+                        TickOffsetOnDeploy = (uint)to,
+                        Child = childdef
+                     }
+                  };
+                  break;
+            }
+            case CustomOrderDefinition.CustomOneofCase.Oco:{
+               return (false,null,"Oco not enabled on api");
+            }
+            case CustomOrderDefinition.CustomOneofCase.Pricer:{
+               return (false,null,"pricer not enabled on api");
+            }
+            case CustomOrderDefinition.CustomOneofCase.Composite:{
+               return (false,null,"Composite not enabled on api");
+            }
+            case CustomOrderDefinition.CustomOneofCase.GlbxIceberg:{
+               if (!WebsocketJsonParsers.TryParse<uint>(custom, "max_show_quantity", out uint? msq, out status) || msq == null) {
+                     return (false, null, status);
+                  }
+                  // mbts.spark.utilities.OrderDataHelper 
+                  customdef = new CustomOrderDefinition() {
+                     GlbxIceberg = new GLBXIcebergOrderDefinition() {
+                        MaxShowQuantity = (uint)msq,
+                     }
+                  };
+                  break;
+            }
+            case CustomOrderDefinition.CustomOneofCase.Splitter:{
+               return (false,null,"splitter not enabled on api");
+            }
+            case CustomOrderDefinition.CustomOneofCase.QueueHolder:{
+               return (false,null,"QueueHolder not enabled on api");
+            }
+            case CustomOrderDefinition.CustomOneofCase.Scratch:{
+               return (false,null,"Scratch not enabled on api");
+            }
+            case CustomOrderDefinition.CustomOneofCase.Prorata:{
+               return (false,null,"Prorata not enabled on api");
+            }
+            case CustomOrderDefinition.CustomOneofCase.DeleteAndCover:{
+               return (false,null,"DeleteAndCover not enabled on api");
+            }
             case CustomOrderDefinition.CustomOneofCase.Limit: {
                   customdef = new CustomOrderDefinition() {
                      Limit = new LimitOrderDefinition()
                   };
                   break;
-               }
+               }       
+            case CustomOrderDefinition.CustomOneofCase.Dynamic: {
+                  if (custom == null) {
+                     return (false, null, "custom not present");
+                  }
+                  if(!WebsocketJsonParsers.TryParse<WebsocketJsonParsers.StringWrapper>(custom, "dynamic_order_type", out WebsocketJsonParsers.StringWrapper? dynamic_order_type, out status) || dynamic_order_type == null ||
+                     !Dynamic_Descriptors.TryGetValue(dynamic_order_type.Value.Val, out var desc)){
+                     return (false, null, status);
+                  }
+                  var def = new DynamicOrderDefinition() {
+                        DynamicOrderType = desc.DynamicOrderType
+                     };
 
-            // case CustomOrderDefinition.CustomOneofCase.Spreader: {
-            //       // if (custom == null) {
-            //       //    message = "custom not present";
-            //       //    return false;
-            //       // }
-            //       //ENTER SPREADER CODE HERE
-            //       break;
-            //    }
-            case CustomOrderDefinition.CustomOneofCase.Iceberg: {
-                  if (custom == null) {
-                     return (false, null, "custom not present");
-                  }
-                  double? sq;
-                  if (!WebsocketJsonParsers.TryParse<double>(custom, "show_quantity", out sq, out status) || sq == null) {
-                     return (false, null, status);
-                  }
-                  customdef = new CustomOrderDefinition() {
-                     Iceberg = new IcebergOrderDefinition() {
-                        ShowQuantity = (double)sq,
-                        Child = new CustomOrderDefinition() {
-                           Limit = new LimitOrderDefinition()
-                        }
+                  foreach (var p in desc.ParamDesc) {
+                     (bool suc, var d, var error)=TryGetDynamicParam(custom,p.ParamName,p,Dynamic_Descriptors);
+                     if(!suc){
+                        return (suc,null,error);
                      }
-                  };
-                  break;
-               }
-            case CustomOrderDefinition.CustomOneofCase.Cover: {
-                  if (custom == null) {
-                     return (false, null, "custom not present");
-                  }
-                  if (!WebsocketJsonParsers.TryParse<uint>(
-                     custom,
-                     "hedge_ticks",
-                     out uint? ht,
-                     out status)
-                     || ht == null) {
-                     return (false, null, status);
-                  }
-                  if (!WebsocketJsonParsers.TryParse<bool>(
-                     custom,
-                     "cancel_on_external_hedge_cancel",
-                     out bool? coehc,
-                     out status)
-                     || coehc == null) {
-                     return (false, null, status);
-                  }
-                  if (!WebsocketJsonParsers.TryParse<bool>(
-                     custom,
-                     "cancel_hedges_on_cancel",
-                     out bool? choc,
-                     out status)
-                     || choc == null) {
-                     return (false, null, status);
-                  }
-                  if (!WebsocketJsonParsers.TryParse<bool>(
-                     custom,
-                     "detach_hedges",
-                     out bool? dh,
-                     out status)
-                     || dh == null) {
-                     return (false, null, status);
+                     def.OrderParams[p.ParamName] = d;
                   }
+
                   customdef = new CustomOrderDefinition() {
-                     Cover = new CoverOrderDefinition() {
-                        Hedge = new CustomOrderDefinition() {
-                           Limit = new LimitOrderDefinition()
-                        },
-                        HedgeTicks = (uint)ht,
-                        CancelOnExternalHedgeCancel = (bool)coehc,
-                        CancelHedgesOnCancel = (bool)choc,
-                        DetachHedges = (bool)dh,
-                        Child = new CustomOrderDefinition() {
-                           Limit = new LimitOrderDefinition()
-                        }
-                     }
+                     Dynamic = def
                   };
                   break;
                }
@@ -124,26 +367,20 @@ namespace bts.websocket_core {
          };
          return (true, customdef, status);
       }
-      private async static Task<(bool, OrderDefinition?, string)> TryGetOrderDefAsync(WebsocketRequest req, IReadOnlyDictionary<uint, SecurityMD> securities) {
+      private async static Task<(bool, OrderDefinition?, string)> TryGetOrderDefAsync(WebsocketRequest req, IReadOnlyDictionary<uint, SecurityMD> securities, IReadOnlyDictionary<string, DynamicOrderDescriptor> Dynamic_Descriptors) {
          if (req.parameters == null) {
             return (false, null, "parameters not present");
          }
-         if (!WebsocketJsonParsers.TryParse<WebsocketJsonParsers.StringWrapper>(
-            req.parameters,
-            "user",
-            out WebsocketJsonParsers.StringWrapper? usr,
-            out string status)
-            || usr == null) {
-            return (false, null, status);
-         }
-         if (!WebsocketJsonParsers.TryParse<WebsocketJsonParsers.StringWrapper>(
-            req.parameters,
-            "account",
-            out WebsocketJsonParsers.StringWrapper? act,
-            out status)
-            || act == null) {
+         if (
+            !WebsocketJsonParsers.TryParse<WebsocketJsonParsers.StringWrapper>(req.parameters,"user",out WebsocketJsonParsers.StringWrapper? usr,out string status) || usr == null ||
+            !WebsocketJsonParsers.TryParse<Venue>(req.parameters, "venue", out Venue? exchange, out status) || exchange == null || 
+            !WebsocketJsonParsers.TryParse<WebsocketJsonParsers.StringWrapper>(req.parameters, "account", out WebsocketJsonParsers.StringWrapper? act, out status) || act == null ||
+            !WebsocketJsonParsers.TryParse<Side>(req.parameters, "side", out Side? side, out status) || side == null || 
+            !WebsocketJsonParsers.TryParse<WebsocketJsonParsers.StringWrapper>(req.parameters, "exchange_security_name", out WebsocketJsonParsers.StringWrapper? exchange_securty_name, out status) || exchange_securty_name == null
+            ) {
             return (false, null, status);
          }
+
          Result<AdminData> adminResult = await _admin.Value.get_admin_data_async(new VoidMessage());
          uint user_id;
          uint account_id;
@@ -160,23 +397,11 @@ namespace bts.websocket_core {
          user_id = (uint)user.Id;
          BTSAccount? account = admin.Accounts.FirstOrDefault(x => x.Name.ToLowerInvariant() == accountname.ToLowerInvariant());
          if (account == null) {
-            return (false, null, "account not corrent");
+            return (false, null, "account not correct");
          }
          account_id = (uint)account.Id;
-         if (!WebsocketJsonParsers.TryParse<Side>(
-            req.parameters,
-            "side",
-            out Side? side,
-            out status)
-            || side == null) {
-            return (false, null, status);
-         }
-         if (!WebsocketJsonParsers.TryParse<WebsocketJsonParsers.StringWrapper>(req.parameters, "exchange_securty_name", out WebsocketJsonParsers.StringWrapper? exchange_securty_name, out status) || exchange_securty_name == null) {
-            return (false, null, status);
-         }
-         if (!WebsocketJsonParsers.TryParse<Venue>(req.parameters, "venue", out Venue? exchange, out status) || exchange == null) {
-            return (false, null, status);
-         }
+         
+
          var _securites = securities;
          if (!_securites.Values.TryGetFirst(sc_md =>
             (
@@ -187,73 +412,100 @@ namespace bts.websocket_core {
             return (false, null, "security not found");
          }
 
-         (var check, var customdef, status) = TryGetCustomDef(req);
+         (var check, var customdef, status) = TryGetCustomDef(req, Dynamic_Descriptors);
          if (check == false) {
             return (false, null, status);
          }
+
+
          return (true, new OrderDefinition {
-            TimeInForce = TimeInForce.Gtc,
+            TimeInForce = TimeInForce.Day,
             SecurityKey = sc_md.id,
             Side = (Side)side,
             UserId = user_id,
             AccountId = account_id,
-            CustomDef = customdef
+            CustomDef = customdef,
+            Account = account.Name,
+            User = user.Name
          }, status);
       }
-      public static async Task<(bool success, string status)> DeleteOrderAsync(WebsocketRequest req, OrdersClient oc) {
-         if (req.parameters == null) {
-            return (false, "parameters not present");
+      static bool TryGetPoints(WebsocketRequest req, [NotNullWhen(true)] out List<PrecomputedTheoPoint>? theos, out string message, IReadOnlyDictionary<uint, SecurityMD> securities) {
+         theos = null;
+         if (!WebsocketJsonParsers.TryParse<WebsocketJsonParsers.PrecomputedTheoPointWrapper>(req.parameters, "points", out WebsocketJsonParsers.PrecomputedTheoPointWrapper[]? wrapper_theos, out message) || wrapper_theos == null) {
+            return false;
          }
-         if (!WebsocketJsonParsers.TryParse<uint>(req.parameters, "order_id", out uint? key, out string status) || key == null) {
-            return (false, status);
+         theos = wrapper_theos.Select(wrapper => {
+            if (WebsocketJsonParsers.PrecomputedTheoPointWrapper.GetTheoPoint(wrapper, out PrecomputedTheoPoint? point, securities)) {
+               return point;
+            }
+            return null;
+         })
+        .Where(p => p != null)
+        .Cast<PrecomputedTheoPoint>()
+        .ToList();
+         return true;
+
+      }
+      public static async Task<(bool success, uint? id, string status)> SetTheosAsync(WebsocketRequest req, OrdersClient oc, IReadOnlyDictionary<uint, SecurityMD> securities) {
+         if (req.parameters == null) {
+            return (false, null, "parameters not present");
+         }
+         var log_txt = $"SetTheosAsync req {req.ToString()}";
+         bts.Log.Gen.warn(log_txt);
+
+         var _securites = securities;
+         if (!TryGetPoints(req, out List<PrecomputedTheoPoint>? theos, out string message, _securites)) {
+            return (false, null, message);
+         }
+         Response res = await oc.TrySetPrecomputedTheosAsync(theos);
+         return (res.ErrorCode != OrderError.Ok, null, res.Msg);
+      }
+      public static async Task<(bool success, uint? id, string status)> DeleteOrderAsync(WebsocketRequest req, OrdersClient oc) {
+         if (req.parameters == null) {
+            return (false, null, "parameters not present");
+         }
+         if (!WebsocketJsonParsers.TryParse<uint>(req.parameters, "order_key", out uint? key, out string status) || key == null) {
+            return (false, null, status);
          }
          Response res = await oc.TryDeleteOrderAsync((uint)key);
          status = res.ErrorCode.ToString();
-         return (res.ErrorCode != OrderError.Ok, status);
+         return (res.ErrorCode != OrderError.Ok, null, status);
       }
-      public static async Task<(bool success, string status)> UpdateOrderAsync(WebsocketRequest request, OrdersClient oc) {
+      public static async Task<(bool success, uint? id, string status)> UpdateOrderAsync(WebsocketRequest request, OrdersClient oc, IReadOnlyDictionary<string, DynamicOrderDescriptor> Dynamic_Descriptors) {
          if (request.parameters == null) {
-            return (false, "parameters not present");
+            return (false, null, "parameters not present");
          }
-         if (!WebsocketJsonParsers.TryParse<uint>(request.parameters, "order_id", out uint? key, out string status) || key == null) {
-            return (false, status);
-         }
-         if (!WebsocketJsonParsers.TryParse(request.parameters, "quantity", out double? quantity, out status) || quantity == null) {
-            return (false, status);
+         if (
+            !WebsocketJsonParsers.TryParse<uint>(request.parameters, "order_key", out uint? key, out string status) || key == null ||
+            !WebsocketJsonParsers.TryParse(request.parameters, "quantity", out double? quantity, out status) || quantity == null ||
+            !WebsocketJsonParsers.TryParse(request.parameters, "price", out double? price, out status) || price == null
+            ) {
+            return (false, null, status);
          }
 
-         if (!WebsocketJsonParsers.TryParse(request.parameters, "price", out double? price, out status) || price == null) {
-            return (false, status);
-         }
-         (var check, var customdef, status) = TryGetCustomDef(request);
+         (var check, var customdef, status) = TryGetCustomDef(request, Dynamic_Descriptors);
          if (check == false || customdef == null) {
-            return (false, status);
+            return (false, null, status);
          }
          Response res = await oc.TryUpdateOrderAsync(customdef, (double)price, (double)quantity, (uint)key);
          status = res.ErrorCode.ToString();
-         return (res.ErrorCode != OrderError.Ok, status);
+         return (res.ErrorCode != OrderError.Ok, null, status);
       }
       public static async Task<(bool success, uint? orderId, string status)> EnterOrderAsync(
-        WebsocketRequest request, OrdersClient ordersClient, IReadOnlyDictionary<uint, SecurityMD> securities) {
-         double? quantity = null;
-         double? price = null;
+        WebsocketRequest request, OrdersClient ordersClient, IReadOnlyDictionary<uint, SecurityMD> securities, IReadOnlyDictionary<string, DynamicOrderDescriptor> Dynamic_Descriptors) {
          OrderDefinition? orderDefinition = null;
          uint? orderId = null;
          string status = string.Empty;
 
-         if (!WebsocketJsonParsers.TryParse(request.parameters, "quantity", out quantity, out status) || quantity == null) {
+         (bool success, orderDefinition, status) = await TryGetOrderDefAsync(request, securities, Dynamic_Descriptors);
+         
+         if (
+            !success || orderDefinition == null ||
+            !WebsocketJsonParsers.TryParse(request.parameters, "quantity", out double? quantity, out status) || quantity == null ||
+            !WebsocketJsonParsers.TryParse(request.parameters, "price", out double? price, out status) || price == null ) {
             return (false, null, status);
          }
 
-         if (!WebsocketJsonParsers.TryParse(request.parameters, "price", out price, out status) || price == null) {
-            return (false, null, status);
-         }
-
-         (bool success, orderDefinition, status) = await TryGetOrderDefAsync(request, securities);
-         if (!success || orderDefinition == null) {
-            return (success, null, status);
-         }
-
          var result = await ordersClient.TryEnterOrderAsync(orderDefinition, (uint)price, (double)quantity);
 
          if (result.ErrorCode != OrderError.Ok) {

src/client/spark_websocket/Core/WebsocketSubscriptionHandler.cs

@@ -100,36 +100,28 @@ namespace bts.websocket_core {
       }
 
       static bool TryGetTradesFilter(WebsocketRequest req, [NotNullWhen(true)] out TradesSubscriptionFilter? trades_filter, out string message) {
-         SecurityCriteria? cri;
          trades_filter = null;
          JObject? filter = req.parameters?["filter"] as JObject;
          if (filter == null) {
             message = "filter not present";
             return false;
          }
-         if (!TryGetFilter(req, out cri, out message)) {
+         if (!TryGetFilter(req, out SecurityCriteria? cri, out message)) {
             return false;
          }
          trades_filter = new TradesSubscriptionFilter(cri);
          return true;
       }
       static bool TryGetFillsFilter(WebsocketRequest req, [NotNullWhen(true)] out FillsSubscriptionFilter? order_filter, out string message) {
-         SecurityCriteria? cri;
          order_filter = null;
-         JObject? filter = req.parameters?["filter"] as JObject;
-         if (filter == null) {
+         if (req.parameters?["filter"] is not JObject filter) {
             message = "filter not present";
             return false;
          }
-         if (!TryGetFilter(req, out cri, out message)) {
-            return false;
-         }
-         uint[]? accounts;
-         if (!WebsocketJsonParsers.TryParse<uint>(filter, "accounts", out accounts, out message)) {
-            return false;
-         }
-         OrderStatus[]? statuses;
-         if (!WebsocketJsonParsers.TryParse<OrderStatus>(filter, "statuses", out statuses, out message)) {
+         if (!TryGetFilter(req, out SecurityCriteria? cri, out message) ||
+            !WebsocketJsonParsers.TryParse<uint>(filter, "accounts", out uint[]? accounts, out message) ||
+            !WebsocketJsonParsers.TryParse<OrderStatus>(filter, "statuses", out OrderStatus[]? statuses, out message)
+            ) {
             return false;
          }
          order_filter = new FillsSubscriptionFilter(
@@ -138,22 +130,16 @@ namespace bts.websocket_core {
          return true;
       }
       static bool TryGetOrdersFilter(WebsocketRequest req, [NotNullWhen(true)] out OrdersSubscriptionFilter? order_filter, out string message) {
-         SecurityCriteria? cri;
          order_filter = null;
-         JObject? filter = req.parameters?["filter"] as JObject;
-         if (filter == null) {
+         if (req.parameters?["filter"] is not JObject filter) {
             message = "filter not present";
             return false;
          }
-         if (!TryGetFilter(req, out cri, out message)) {
-            return false;
-         }
-         uint[]? accounts;
-         if (!WebsocketJsonParsers.TryParse<uint>(filter, "accounts", out accounts, out message)) {
-            return false;
-         }
-         OrderStatus[]? statuses;
-         if (!WebsocketJsonParsers.TryParse<OrderStatus>(filter, "statuses", out statuses, out message)) {
+         if (
+            !TryGetFilter(req, out SecurityCriteria? cri, out message) ||
+            !WebsocketJsonParsers.TryParse<uint>(filter, "accounts", out uint[]? accounts, out message) ||
+            !WebsocketJsonParsers.TryParse<OrderStatus>(filter, "statuses", out OrderStatus[]? statuses, out message)
+            ) {
             return false;
          }
          order_filter = new OrdersSubscriptionFilter(
@@ -165,39 +151,19 @@ namespace bts.websocket_core {
 
       static bool TryGetFilter(WebsocketRequest req, [NotNullWhen(true)] out SecurityCriteria? cri, out string message) {
          cri = null;
-         JObject? filter = req.parameters?["filter"] as JObject;
-         if (filter == null) {
+         if (req.parameters?["filter"] is not JObject filter) {
             message = "filter not present";
             return false;
          }
-         Venue[]? exchanges;
-         if (!WebsocketJsonParsers.TryParse<Venue>(filter, "venue", out exchanges, out message)) {
-            return false;
-         }
-         SecurityType[]? types;
-         if (!WebsocketJsonParsers.TryParse<SecurityType>(filter, "kind", out types, out message)) {
-            return false;
-         }
-         PutOrCall[]? callputs;
-         if (!WebsocketJsonParsers.TryParse<PutOrCall>(filter, "call_or_put", out callputs, out message)) {
-            return false;
-         }
-         DateTime[]? expiration;
-         if (!WebsocketJsonParsers.TryParse<DateTime>(filter, "expiration", out expiration, out message)) {
-            return false;
-         }
-         // string[]? symbols = (string[]?)(filter.GetValueOrDefault("symbol")?.ToString()?.Split(",").Select(s => s.ToUpper()));
-         WebsocketJsonParsers.ListWrapper? symbols;
-         if (!WebsocketJsonParsers.TryParse<WebsocketJsonParsers.ListWrapper>(filter, "symbol", out symbols, out message)) {
-            return false;
-         }
-         WebsocketJsonParsers.ListWrapper? names;
-         if (!WebsocketJsonParsers.TryParse<WebsocketJsonParsers.ListWrapper>(filter, "name", out names, out message)) {
-            return false;
-         }
-         // filter.GetValueOrDefault("symbol")?.ToString()?.Split(",");
-         double[]? strikes;
-         if (!WebsocketJsonParsers.TryParse<double>(filter, "strikes", out strikes, out message)) {
+         if (
+            !WebsocketJsonParsers.TryParse<Venue>(filter, "venue", out Venue[]? exchanges, out message) ||
+            !WebsocketJsonParsers.TryParse<SecurityType>(filter, "kind", out SecurityType[]? types, out message) ||
+            !WebsocketJsonParsers.TryParse<PutOrCall>(filter, "call_or_put", out PutOrCall[]? callputs, out message) ||
+            !WebsocketJsonParsers.TryParse<DateTime>(filter, "expiration", out DateTime[]? expiration, out message) || 
+            !WebsocketJsonParsers.TryParse<WebsocketJsonParsers.StringListWrapper>(filter, "symbol", out WebsocketJsonParsers.StringListWrapper? symbols, out message) ||
+            !WebsocketJsonParsers.TryParse<WebsocketJsonParsers.StringListWrapper>(filter, "name", out WebsocketJsonParsers.StringListWrapper? names, out message) ||
+            !WebsocketJsonParsers.TryParse<double>(filter, "strikes", out double[]? strikes, out message)
+         ) {
             return false;
          }
          var clause = new SecurityCriteriaClause(
@@ -223,13 +189,12 @@ namespace bts.websocket_core {
          out string status) {
          hdlr = null;
 
-         SUBSCRIPTION_TYPE? type;
-         if (!WebsocketJsonParsers.TryParse<SUBSCRIPTION_TYPE>(req.parameters, "type", out type, out status) || status != "") {
-            return false;
-         }
-         if (!WebsocketJsonParsers.TryParse<WebsocketJsonParsers.StringWrapper>(req.parameters, "subscription_id", out WebsocketJsonParsers.StringWrapper? id, out status) || id == null) {
+         if (
+            !WebsocketJsonParsers.TryParse<SUBSCRIPTION_TYPE>(req.parameters, "type", out SUBSCRIPTION_TYPE? type, out status) || status != "" ||
+            !WebsocketJsonParsers.TryParse<WebsocketJsonParsers.StringWrapper>(req.parameters, "subscription_id", out WebsocketJsonParsers.StringWrapper? id, out status) || id == null) {
             return false;
          }
+        
          Action<WebsocketSubscriptionHandler> sub;
          switch (type) {
             case SUBSCRIPTION_TYPE.BOOKS: {

src/data_files/documents/order_flow.txt

@@ -79,6 +79,24 @@ Misc:
 To search for a publisher for an event search for make_stream<Event_Name>, for 
 example make_stream<OrderStateUpdateEvent>()
 
+is_gateway_connected can be false in the NativeOrder class (GlbxNativeOrder) ->
+OrderServices->is_gateway_connected->
+SessionServices->is_gateway_connected
+SessionServicesData->gw_connected
+OrderManager sets SessionServicesData->gw_connected on void OrderManager::on_gateway_state(const mercury::Update<GatewayStatusChangeEvent> &m)
+
+
+Exchange gateway is down (debugging):
+-------------------------------------
+This happens at times and if often a config issue.
+GlbCNativeOrder::_on_activate ->
+OrderServices::is_gateway_connected ->
+SessionServices::is_gateway_connected ->
+SessionServicesData::
+
+IExecutionGateway::on_gw_status_change is what sends out the status
+
+
 --------------------------------------------------------------------------
 market data
 --------------------------------------------------------------------------

src/lib/api/md/GlbxMarketDataConfig.proto

@@ -6,17 +6,19 @@ package bts.api.md;
 
 option csharp_namespace = "bts.api.md";
 
+message GlbxActivePlatform {
+  string host = 1;
+  string user = 2;
+  string password = 3;
+}
+message GlbxOnixPlatform {
+  string temp =1; // will add everything related to onix here.
+}
 
 message GlbxMarketDataConfig {
-  uint32 unique_check_value = 1;
-  //uint32 num_depth_levels = 1;
-  //uint32 depth_throttle_ms = 2;
-  bts.api.domain.SecurityCriteria security_criteria = 3;
-  // Market data isn't subscribed for securities listed here (even if they are
-  // included by 'security_criteria'). There are some securities that don't
-  // fit into our system just yet, such as 'BTCBIDR', which trades in prices
-  // larger than our cpp 'domain::Price' class supports.
-  //repeated string excluded_securities = 4;
-  //string url_rest_base = 5;
-  //string url_ws_base = 6;
+  bts.api.domain.SecurityCriteria security_criteria = 2;
+  oneof Platform {
+    GlbxActivePlatform glbx_active_platform = 10;
+    GlbxOnixPlatform glbx_onix_platform = 11;
+  }
 }

src/lib/cmake/GetLinuxClientSources.cmake

@@ -93,10 +93,6 @@ function(get_cpp_sources dest)
     md/MDConsumerInterop.cpp
     md/MDProducer.cpp
     md/MDProducerHandle.cpp
-    md/MarketDataError.cpp
-    md/MarketDataGateway.cpp
-    md/MarketDataRegistrationFactory.cpp
-    md/MarketDataSubscription.cpp
     md/MemorySegmentUtils.cpp
     md/PriceSize.cpp
     md/Product.cpp
@@ -113,8 +109,6 @@ function(get_cpp_sources dest)
     md/cme/MDP3Utils.cpp
     md/common.cpp
     md/ice/ICEUtils.cpp
-    md/recording/MDRecorder.cpp
-    md/recording/MDReplayer.cpp
     md/relay/QueryServer.cpp
     md/relay/RelayClient.cpp
     md/relay/RelayRecorder.cpp

src/lib/cmake/GetLinuxServerLinkLibraries.cmake

@@ -25,8 +25,23 @@ function(get_link_libraries dest)
     Packet++
     Common++
     ssh2
+    ACE-5.7.5
+    ActivBase_x86-64_ubuntu22-04_gcc123_mdid
+    ActivOneApi_x86-64_ubuntu22-04_gcc123_mdid
+    ActivAgentApplication_x86-64_ubuntu22-04_gcc123_mdid
+    ActivDatabase_x86-64_ubuntu22-04_gcc123_mdid
+    ActivFramework_x86-64_ubuntu22-04_gcc123_mdid
+    ActivServices_x86-64_ubuntu22-04_gcc123_mdid
+    BoostAtomic_x86-64_ubuntu22-04_gcc123_mdid
+    BoostChrono_x86-64_ubuntu22-04_gcc123_mdid
+    BoostProgramOptions_x86-64_ubuntu22-04_gcc123_mdid
+    BoostFilesystem_x86-64_ubuntu22-04_gcc123_mdid
+    -lm
+    -lgcc_s
+    -lc
     -lrt
     -ldl
+    -lstdc++
     -lpcap
     -static-libgcc
     -static-libstdc++

src/lib/cmake/GetWindowsClientSources.cmake

@@ -87,10 +87,6 @@ function(get_cpp_sources dest)
     md/MDConsumerInterop.cpp
     md/MDProducer.cpp
     md/MDProducerHandle.cpp
-    md/MarketDataError.cpp
-    md/MarketDataGateway.cpp
-    md/MarketDataRegistrationFactory.cpp
-    md/MarketDataSubscription.cpp
     md/MemorySegmentUtils.cpp
     md/PriceSize.cpp
     md/Product.cpp

src/lib/config/profiles/default/jetfix.conf

@@ -3,7 +3,7 @@ JetFIXSimConnection {
    value {
       sim {
         name: "sim"
-        venue: VENUE_SIM
+        venue: VENUE_GLBX
         socket_connect_host_ip: "127.0.0.1"
         socket_connect_port: @Ports.glbx_sim_fix_port
         sender_compid: "SPARK" # this is the FIX session name
@@ -17,7 +17,9 @@ JetFIXSimConnection {
         raw_data: "1JE9X"
         use_cme_secure_logon: false
         seconds_to_pause : 1
-	max_connection_attempts: 9999999
+        max_order_msgs_per_window: 300
+        window_duration_msecs: 1000
+	     max_connection_attempts: 9999999
       }
       sim_ice {
         name: "sim_ice"

src/lib/core/EnumMacros.h

@@ -1,5 +1,7 @@
 #pragma once
 
+#include <ostream>
+
 #define ENUM_BASIC_MEMBERS(type)                                               \
    Value value;                                                                \
                                                                                \

src/lib/core/TimeStamp.cpp

@@ -23,7 +23,18 @@ TimeStamp::TimeStamp(uint32_t year, uint32_t month, uint32_t day) {
    value_ =
        std::chrono::duration_cast<std::chrono::nanoseconds>(duration).count();
 }
+bool TimeStamp::from_ymd(
+    std::string yearMonthDay, TimeStamp *result) {
+   uint32_t year, month, day;
+   std::istringstream ss(yearMonthDay);
+    char dash; // to consume the dash character in the input
 
+    // Use stringstream to extract year, month, and day from the input string
+    ss >> year >> dash >> month >> dash >> day;
+    if(ss.fail())
+      return false;
+    return TimeStamp::from_ymd(year,month,day,result);
+}
 bool TimeStamp::from_ymd(
     std::uint32_t year, std::uint32_t month, std::uint32_t day, TimeStamp *result) {
    if (!is_valid_ymd(year, month, day)) {

src/lib/core/TimeStamp.h

@@ -41,6 +41,9 @@ struct TimeStamp {
        std::uint32_t month,
        std::uint32_t day,
        TimeStamp *result);
+   static bool from_ymd(
+      std::string yearMonthDay,
+       TimeStamp *result);
 
    constexpr std::uint64_t nanos() const;
    constexpr std::uint64_t micros() const;

src/lib/domain_model/SecurityKey.h

@@ -1,5 +1,14 @@
 #pragma once
+
+#include "lib/core/FMT_IMPL.h"
 #include "lib/utils/ValueWrapper.h"
+
+#include <cstdint>
+
 namespace bts {
-using SecurityKey = utils::ValueWrapper<struct SecurityKeyTag, uint32_t>;
-}
+
+using SecurityKey = utils::ValueWrapper<struct SecurityKeyTag, std::uint32_t>;
+
+} // namespace bts
+
+FMT_IMPL(bts::SecurityKey, k, "{}", k.value());

src/lib/io/websocket/JsonRpcRequest.h

@@ -69,8 +69,6 @@ inline void from_json(const nlohmann::json &j, JsonRpcRequestT<ParamsT> &req) {
    j.at("method").get_to(req.method);
    j.at("params").get_to(req.params);
    std::vector<std::string> vec;
-   *vec.begin();
-   vec.begin()->size();
    auto id_j = j.find("id");
    if (id_j == j.end()) {
       req.id = std::nullopt;

src/lib/io/websocket/WebsocketMessage.h

@@ -1,6 +1,7 @@
 #pragma once
+
+#include <cstdint>
 #include <ostream>
-#include <string>
 #include <vector>
 
 namespace bts::io::websocket {