Reversehedging fixed

.devcontainer/Dockerfile

@@ -65,7 +65,7 @@ RUN \
    && apt-get install -y cmake
 
 # Dotnet
-RUN apt-get update && apt-get install -y dotnet8
+RUN apt-get update && apt-get install -y dotnet7
 
 # Node
 ARG node_major=18

.gitignore

@@ -60,4 +60,3 @@ trace_data/
 *.Generated.cs
 *.lst
 .env.local
-.secrets

.vscode/launch.json

@@ -1,14 +1,6 @@
 {
   "version": "0.2.0",
   "configurations": [
-    {
-      "name": "Python Debugger: Current File with Arguments",
-      "type": "debugpy",
-      "request": "launch",
-      "program": "${file}",
-      "console": "integratedTerminal",
-      "args": "${command:pickArgs}"
-    },
     {
       "name": ".NET Core Attach",
       "type": "coreclr",
@@ -30,7 +22,7 @@
         "--config",
         "${workspaceFolder}/src/DataStreamSuite/DataStreamPublisher/Configurations/${input:dataStreamPublisherConfig}.json"
       ],
-      "cwd": "build/Debug/bin/net8.0",
+      "cwd": "build/Debug/bin/net7.0",
       "stopAtEntry": false,
       "console": "integratedTerminal"
     },
@@ -90,7 +82,7 @@
           "--config",
           "${workspaceFolder}/src/DataStreamSuite/DataStreamQueryAnalyzer/Configurations/DataStreamQueryAnalyzer.json"
         ],
-        "cwd": "build/Debug/bin/net8.0-windows",
+        "cwd": "build/Debug/bin/net7.0-windows",
         "env": {
           "PYTHON_EXE": "${workspaceFolder}/.venv/Scripts/python.exe"
         },
@@ -114,7 +106,7 @@
         "--config",
         "${workspaceFolder}/src/DataStreamSuite/PubSubServer/Configurations/${input:pubSubServerConfig}.json"
       ],
-      "cwd": "build/Debug/bin/net8.0",
+      "cwd": "build/Debug/bin/net7.0",
       "stopAtEntry": false,
       "console": "integratedTerminal"
     },
@@ -126,7 +118,7 @@
       "windows": {
         "preLaunchTask": "Build Spark (Debug)",
         "program": "Spark.exe",
-        "cwd": "build/Debug/bin/net8.0-windows",
+        "cwd": "build/Debug/bin/net7.0-windows",
       },
       "stopAtEntry": false,
       "console": "integratedTerminal"
@@ -138,7 +130,7 @@
       "program": "",
       "windows": {
         "program": "Spark.exe",
-        "cwd": "build/Debug/bin/net8.0-windows",
+        "cwd": "build/Debug/bin/net7.0-windows",
       },
       "stopAtEntry": false,
       "console": "integratedTerminal"
@@ -155,7 +147,7 @@
           "--sparkd",
           "${input:sparkConfigServer}"
         ],
-        "cwd": "build/Debug/bin/net8.0-windows",
+        "cwd": "build/Debug/bin/net7.0-windows",
       },
       "stopAtEntry": false,
       "console": "integratedTerminal"
@@ -168,7 +160,7 @@
       "windows": {
         "preLaunchTask": "Build Spark (Debug)",
         "program": "Spark.Console.exe",
-        "cwd": "build/Debug/bin/net8.0",
+        "cwd": "build/Debug/bin/net7.0",
         "args": [
           "test",
           "${input:sparkTest}"
@@ -184,7 +176,7 @@
       "program": "",
       "windows": {
         "program": "Spark.Console.exe",
-        "cwd": "build/Debug/bin/net8.0",
+        "cwd": "build/Debug/bin/net7.0",
         "args": [
           "test",
           "${input:sparkTest}"
@@ -334,4 +326,4 @@
       "description": "Test to run",
     },
   ]
-}
+}

CMakePresets.json

@@ -32,8 +32,8 @@
         "CMAKE_TOOLCHAIN_FILE": "${sourceDir}/vcpkg/scripts/buildsystems/vcpkg.cmake",
         "EXTERNALS_ROOT": "${sourceDir}/externals/windows",
         "CONFIG_DEFAULTS_PATH": "config/profiles/default/*.conf",
-        "DOTNET_BINARY_DIR": "${sourceDir}/build/$<CONFIG>/bin/net8.0",
-        "DOTNET_WINDOWS_BINARY_DIR": "${sourceDir}/build/$<CONFIG>/bin/net8.0-windows",
+        "DOTNET_BINARY_DIR": "${sourceDir}/build/$<CONFIG>/bin/net7.0",
+        "DOTNET_WINDOWS_BINARY_DIR": "${sourceDir}/build/$<CONFIG>/bin/net7.0-windows",
         "EXPORT_MD_CONSUMER": {
           "type": "BOOL",
           "value": "ON"
@@ -64,11 +64,6 @@
         "CMAKE_INSTALL_RPATH": ".",
         "CMAKE_INSTALL_PREFIX": "${sourceDir}/install/${presetName}",
         "CMAKE_EXE_LINKER_FLAGS": "-Wno-undef",
-        "CMAKE_TOOLCHAIN_FILE": "${sourceDir}/vcpkg/scripts/buildsystems/vcpkg.cmake",
-        "DATABENTO_USE_EXTERNAL_JSON": {
-          "type": "BOOL",
-          "value": "ON"
-        },
         "EXTERNALS_ROOT": "${sourceDir}/externals/linux",
         "BUILD_SDK": {
           "type": "BOOL",
@@ -187,7 +182,7 @@
         "CMAKE_CXX_FLAGS_RELEASE": "-fPIC -Wall -pedantic -Wextra -Werror -Wno-missing-braces -Wno-ignored-qualifiers --std=c++20 -fdiagnostics-color -fno-omit-frame-pointer -DNDEBUG -O3",
         "CMAKE_INSTALL_PREFIX": "${sourceDir}/install/${presetName}",
         "CONFIG_DEFAULTS_PATH": "config/profiles/default/*.conf",
-        "DOTNET_BINARY_DIR": "${sourceDir}/build/$<CONFIG>/bin/net8.0",
+        "DOTNET_BINARY_DIR": "${sourceDir}/build/$<CONFIG>/bin/net7.0",
         "EXPORT_MD_CONSUMER": {
           "type": "BOOL",
           "value": "ON"

compose.yaml

@@ -33,7 +33,6 @@ services:
       resources:
         limits:
           memory: "${SPARKD__MEMORY:-4gb}"
-
   sparkd-symbols:
     volumes:
       - ${SPARKD__DATA_DIR:-sparkd_data}:/home/bts/sparkdata/data
@@ -47,7 +46,7 @@ services:
     shm_size: "${SPARKD__SHM_SIZE:-512mb}"
     command: >
       sh -c "
-        echo bts | sudo -S chown -R bts:bts /home/bts/sparkdata
+        echo bts | sudo -S chown -R bts:bts /home/bts/sparkdata/data
         if [ -e "/home/bts/sparkdata/data/logging.sparkd/log" ]; then
           mv "/home/bts/sparkdata/data/logging.sparkd/log" "/home/bts/sparkdata/data/logging.sparkd/log_$(date '+%Y%m%d-%H%M')"
         fi
@@ -68,9 +67,6 @@ services:
       - "${SPARKD__QUICKBUS_PORT}:${SPARKD__QUICKBUS_PORT}"
       - "${SPARKD__SIM_FIX_PORT}:${SPARKD__SIM_FIX_PORT}"
     deploy:
-      restart_policy:
-        condition: any
-        delay: 3s
       resources:
         limits:
           memory: "${SPARKD__MEMORY:-4gb}"

src/DataStreamSuite/ArithmeticExpressions/ArithmeticExpressions.csproj

@@ -1,7 +1,7 @@
 <Project Sdk="Microsoft.NET.Sdk">
 
   <PropertyGroup>
-    <TargetFramework>net8.0</TargetFramework>
+    <TargetFramework>net7.0</TargetFramework>
     <ImplicitUsings>enable</ImplicitUsings>
     <Nullable>enable</Nullable>
   </PropertyGroup>

src/DataStreamSuite/BaseUtilities/BaseUtilities.csproj

@@ -1,7 +1,7 @@
 <Project Sdk="Microsoft.NET.Sdk">
 
   <PropertyGroup>
-    <TargetFramework>net8.0</TargetFramework>
+    <TargetFramework>net7.0</TargetFramework>
     <ImplicitUsings>enable</ImplicitUsings>
     <Nullable>enable</Nullable>
   </PropertyGroup>

src/DataStreamSuite/BlueRegistry/BlueRegistry.csproj

@@ -1,7 +1,7 @@
 <Project Sdk="Microsoft.NET.Sdk">
 
   <PropertyGroup>
-    <TargetFramework>net8.0</TargetFramework>
+    <TargetFramework>net7.0</TargetFramework>
     <ImplicitUsings>enable</ImplicitUsings>
     <Nullable>enable</Nullable>
   </PropertyGroup>

src/DataStreamSuite/BooleanExpressions/BooleanExpressions.csproj

@@ -1,14 +1,14 @@
 <Project Sdk="Microsoft.NET.Sdk">
 
   <PropertyGroup>
-    <TargetFramework>net8.0</TargetFramework>
+    <TargetFramework>net7.0</TargetFramework>
     <ImplicitUsings>enable</ImplicitUsings>
     <Nullable>enable</Nullable>
   </PropertyGroup>
 
   <ItemGroup>
     <PackageReference Include="Google.Protobuf" Version="3.24.3" />
-    <PackageReference Include="YaccLexTools" Version="1.2.3" />
+    <PackageReference Include="YaccLexTools" Version="1.2.0" />
   </ItemGroup>
 
   <ItemGroup>

src/DataStreamSuite/BooleanExpressions/GplexBuffers.cs

@@ -24,6 +24,8 @@ namespace QUT.GplexBuffers
         public BufferException(string message) : base(message) { }
         public BufferException(string message, Exception innerException)
             : base(message, innerException) { }
+        protected BufferException(SerializationInfo info, StreamingContext context)
+            : base(info, context) { }
     }
 
     internal abstract class ScanBuff
@@ -58,10 +60,12 @@ namespace QUT.GplexBuffers
             return new BuildBuffer(source);
         }
 
+#if (!BYTEMODE)
         public static ScanBuff GetBuffer(Stream source, int fallbackCodePage)
         {
             return new BuildBuffer(source, fallbackCodePage);
         }
+#endif // !BYTEMODE
 #endif // !NOFILES
     }
 
@@ -349,15 +353,17 @@ namespace QUT.GplexBuffers
         {
             FileStream fStrm = (stream as FileStream);
             if (fStrm != null) FileName = fStrm.Name;
-            NextBlk = BlockReaderFactory.Get(stream);
+            NextBlk = BlockReaderFactory.Raw(stream);
         }
 
+#if (!BYTEMODE)
         public BuildBuffer(Stream stream, int fallbackCodePage)
         {
             FileStream fStrm = (stream as FileStream);
             if (fStrm != null) FileName = fStrm.Name;
             NextBlk = BlockReaderFactory.Get(stream, fallbackCodePage);
         }
+#endif
 
         /// <summary>
         /// Marks a conservative lower bound for the buffer,
@@ -428,19 +434,43 @@ namespace QUT.GplexBuffers
     //
     internal static class BlockReaderFactory
     {
+        public static BlockReader Raw(Stream stream)
+        {
+            return delegate(char[] block, int index, int number)
+            {
+                byte[] b = new byte[number];
+                int count = stream.Read(b, 0, number);
+                int i = 0;
+                int j = index;
+                for (; i < count; i++, j++)
+                    block[j] = (char)b[i];
+                return count;
+            };
+        }
 
-        public static BlockReader Get(Stream stream, int fallbackCodePage = -2)
+#if (!BYTEMODE)
+        public static BlockReader Get(Stream stream, int fallbackCodePage)
         {
             Encoding encoding;
             int preamble = Preamble(stream);
 
             if (preamble != 0)  // There is a valid BOM here!
                 encoding = Encoding.GetEncoding(preamble);
-            else if (fallbackCodePage >= 0) // Anything but "guess"
+            else if (fallbackCodePage == -1) // Fallback is "raw" bytes
+                return Raw(stream);
+            else if (fallbackCodePage != -2) // Anything but "guess"
                 encoding = Encoding.GetEncoding(fallbackCodePage);
             else // This is the "guess" option
-                encoding = GuessEncoding(stream);
-
+            {
+                int guess = new Guesser(stream).GuessCodePage();
+                stream.Seek(0, SeekOrigin.Begin);
+                if (guess == -1) // ==> this is a 7-bit file
+                    encoding = Encoding.ASCII;
+                else if (guess == 65001)
+                    encoding = Encoding.UTF8;
+                else             // ==> use the machine default
+                    encoding = Encoding.Default;
+            }
             StreamReader reader = new StreamReader(stream, encoding);
             return reader.Read;
         }
@@ -458,14 +488,6 @@ namespace QUT.GplexBuffers
             int b2 = stream.ReadByte();
             if (b0 == 0xef && b1 == 0xbb && b2 == 0xbf)
                 return 65001; // UTF8
-
-            int b3 = stream.ReadByte();
-            // Check for UTF-32 BOM (Byte Order Mark)
-            if (b0 == 0x00 && b1 == 0x00 && b2 == 0xFE && b3 == 0xFF)
-                return 12001; // UTF-32BE
-            if (b0 == 0xFF && b1 == 0xFE && b2 == 0x00 && b3 == 0x00)
-                return 12000; // UTF-32LE
-
             //
             // There is no unicode preamble, so we
             // return denoter for the machine default.
@@ -473,150 +495,7 @@ namespace QUT.GplexBuffers
             stream.Seek(0, SeekOrigin.Begin);
             return 0;
         }
-
-        private static Encoding GuessEncoding(Stream stream)
-        {
-            stream.Seek(0, SeekOrigin.Begin);
-            if (IsUtf16(stream, out Encoding encoding))
-            {
-                stream.Seek(0, SeekOrigin.Begin);
-                return encoding;
-            }
-
-            stream.Seek(0, SeekOrigin.Begin);
-            if (IsUtf8(stream))
-            {
-                stream.Seek(0, SeekOrigin.Begin);
-                return Encoding.UTF8;
-            }
-
-            stream.Seek(0, SeekOrigin.Begin);
-            if (IsAscii(stream))
-            {
-                stream.Seek(0, SeekOrigin.Begin);
-                return Encoding.ASCII;
-            }
-
-#if !NET462_OR_GREATER
-            stream.Seek(0, SeekOrigin.Begin);
-            if (IsLatin1(stream))
-            {
-                stream.Seek(0, SeekOrigin.Begin);
-                return Encoding.Latin1;
-            }
-#endif
-            stream.Seek(0, SeekOrigin.Begin);
-            return Encoding.Default;
-        }
-
-        private static bool IsUtf8(Stream stream)
-        {
-            int b;
-            while ((b = stream.ReadByte()) != -1)
-            {
-                if ((b & 0x80) == 0) // 0xxxxxxx
-                {
-                    continue;
-                }
-                else if ((b & 0xE0) == 0xC0) // 110xxxxx 10xxxxxx
-                {
-                    if (!IsValidUtf8ContinuationByte(stream)) return false;
-                }
-                else if ((b & 0xF0) == 0xE0) // 1110xxxx 10xxxxxx 10xxxxxx
-                {
-                    if (!IsValidUtf8ContinuationByte(stream) || !IsValidUtf8ContinuationByte(stream)) return false;
-                }
-                else if ((b & 0xF8) == 0xF0) // 11110xxx 10xxxxxx 10xxxxxx 10xxxxxx
-                {
-                    if (!IsValidUtf8ContinuationByte(stream) || !IsValidUtf8ContinuationByte(stream) || !IsValidUtf8ContinuationByte(stream)) return false;
-                }
-                else
-                {
-                    return false;
-                }
-            }
-            return true;
-        }
-
-        private static bool IsValidUtf8ContinuationByte(Stream stream)
-        {
-            int b = stream.ReadByte();
-            return b != -1 && (b & 0xC0) == 0x80; // 10xxxxxx
-        }
-
-        private static bool IsUtf16(Stream stream, out Encoding encoding)
-        {
-            encoding = Encoding.Default;
-
-            if (stream.Length >= 2)
-            {
-                // Read the first two bytes
-                int b0 = stream.ReadByte();
-                int b1 = stream.ReadByte();
-
-                // Check for UTF-16 BOM (Byte Order Mark)
-                if (b0 == 0xFF && b1 == 0xFE)
-                {
-                    encoding = Encoding.Unicode;
-                    return true; // UTF-16LE
-                }
-                if (b0 == 0xFE && b1 == 0xFF)
-                {
-                    encoding = Encoding.BigEndianUnicode;
-                    return true; // UTF-16BE
-                }
-
-                // Reset the stream position
-                stream.Seek(0, SeekOrigin.Begin);
-
-                // Check for common UTF-16 patterns
-                byte[] buffer = new byte[2];
-                while (stream.Read(buffer, 0, 2) == 2)
-                {
-                    if (buffer[0] == 0x00 && buffer[1] != 0x00)
-                    {
-                        encoding = Encoding.BigEndianUnicode;
-                        return true; // Likely UTF-16BE
-                    }
-                    if (buffer[0] != 0x00 && buffer[1] == 0x00)
-                    {
-                        encoding = Encoding.Unicode;
-                        return true; // Likely UTF-16LE
-                    }
-                }
-            }
-
-            return false;
-        }
-
-        private static bool IsAscii(Stream stream)
-        {
-            int b;
-            while ((b = stream.ReadByte()) != -1)
-            {
-                // Latin1 includes byte values from 0x00 to 0xFF
-                if (b > 0x7F)
-                {
-                    return false;
-                }
-            }
-            return true;
-        }
-
-        private static bool IsLatin1(Stream stream)
-        {
-            int b;
-            while ((b = stream.ReadByte()) != -1)
-            {
-                // Latin1 includes byte values from 0x00 to 0xFF
-                if (b > 0xFF)
-                {
-                    return false;
-                }
-            }
-            return true;
-        }
-
+#endif // !BYTEMODE
     }
 #endif // !NOFILES
     #endregion Buffer classes
@@ -661,6 +540,180 @@ namespace QUT.GplexBuffers
             return 0;
         }
     }
+#region guesser
+#if (!BYTEMODE)
+    // ==============================================================
+    // ============          Encoding Guesser           =============
+    // ==============================================================
+
+    /// <summary>
+    /// This class provides a simple finite state automaton that
+    /// scans the file looking for (1) valid UTF-8 byte patterns,
+    /// (2) bytes >= 0x80 which are not part of a UTF-8 sequence.
+    /// The method then guesses whether it is UTF-8 or maybe some 
+    /// local machine default encoding.  This works well for the
+    /// various Latin encodings.
+    /// </summary>
+    internal class Guesser
+    {
+        ScanBuff buffer;
+
+        public int GuessCodePage() { return Scan(); }
+
+        const int maxAccept = 10;
+        const int initial = 0;
+        const int eofNum = 0;
+        const int goStart = -1;
+        const int INITIAL = 0;
+        const int EndToken = 0;
+
+        #region user code
+        /* 
+         *  Reads the bytes of a file to determine if it is 
+         *  UTF-8 or a single-byte code page file.
+         */
+        public long utfX;
+        public long uppr;
+        #endregion user code
+
+        int state;
+        int currentStart = startState[0];
+        int code;
+
+        #region ScannerTables
+        static int[] startState = new int[] { 11, 0 };
+
+        #region CharacterMap
+        static sbyte[] map = new sbyte[256] {
+/*     '\0' */ 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 
+/*   '\x10' */ 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 
+/*   '\x20' */ 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 
+/*      '0' */ 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 
+/*      '@' */ 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 
+/*      'P' */ 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 
+/*      '`' */ 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 
+/*      'p' */ 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 
+/*   '\x80' */ 2, 2, 2, 2, 2, 2, 2, 2, 2, 2, 2, 2, 2, 2, 2, 2, 
+/*   '\x90' */ 2, 2, 2, 2, 2, 2, 2, 2, 2, 2, 2, 2, 2, 2, 2, 2, 
+/*   '\xA0' */ 2, 2, 2, 2, 2, 2, 2, 2, 2, 2, 2, 2, 2, 2, 2, 2, 
+/*   '\xB0' */ 2, 2, 2, 2, 2, 2, 2, 2, 2, 2, 2, 2, 2, 2, 2, 2, 
+/*   '\xC0' */ 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 
+/*   '\xD0' */ 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 
+/*   '\xE0' */ 3, 3, 3, 3, 3, 3, 3, 3, 3, 3, 3, 3, 3, 3, 3, 3, 
+/*   '\xF0' */ 4, 4, 4, 4, 4, 4, 4, 4, 5, 5, 5, 5, 5, 5, 5, 5 };
+        #endregion
+
+        static sbyte[][] nextState = new sbyte[][] {
+            new sbyte[] {0, 0, 0, 0, 0, 0},
+            new sbyte[] {-1, -1, 10, -1, -1, -1},
+            new sbyte[] {-1, -1, -1, -1, -1, -1},
+            new sbyte[] {-1, -1, 8, -1, -1, -1},
+            new sbyte[] {-1, -1, 5, -1, -1, -1},
+            new sbyte[] {-1, -1, 6, -1, -1, -1},
+            new sbyte[] {-1, -1, 7, -1, -1, -1},
+            null,
+            new sbyte[] {-1, -1, 9, -1, -1, -1},
+            null,
+            null,
+            new sbyte[] {-1, 1, 2, 3, 4, 2}
+        };
+
+
+        [SuppressMessage("Microsoft.Performance", "CA1810:InitializeReferenceTypeStaticFieldsInline")]
+        // Reason for suppression: cannot have self-reference in array initializer.
+        static Guesser()
+        {
+            nextState[7] = nextState[2];
+            nextState[9] = nextState[2];
+            nextState[10] = nextState[2];
+        }
+
+        int NextState()
+        {
+            if (code == ScanBuff.EndOfFile)
+                return eofNum;
+            else
+                return nextState[state][map[code]];
+        }
+        #endregion
+
+        public Guesser(System.IO.Stream file) { SetSource(file); }
+
+        public void SetSource(System.IO.Stream source)
+        {
+            this.buffer = new BuildBuffer(source);
+            code = buffer.Read();
+        }
+
+        int Scan()
+        {
+            for (; ; )
+            {
+                int next;
+                state = currentStart;
+                while ((next = NextState()) == goStart)
+                    code = buffer.Read();
+
+                state = next;
+                code = buffer.Read();
+
+                while ((next = NextState()) > eofNum)
+                {
+                    state = next;
+                    code = buffer.Read();
+                }
+                if (state <= maxAccept)
+                {
+                    #region ActionSwitch
+#pragma warning disable 162
+                    switch (state)
+                    {
+                        case eofNum:
+                            switch (currentStart)
+                            {
+                                case 11:
+                                    if (utfX == 0 && uppr == 0) return -1; /* raw ascii */
+                                    else if (uppr * 10 > utfX) return 0;   /* default code page */
+                                    else return 65001;                     /* UTF-8 encoding */
+                                    break;
+                            }
+                            return EndToken;
+                        case 1: // Recognized '{Upper128}',	Shortest string "\xC0"
+                        case 2: // Recognized '{Upper128}',	Shortest string "\x80"
+                        case 3: // Recognized '{Upper128}',	Shortest string "\xE0"
+                        case 4: // Recognized '{Upper128}',	Shortest string "\xF0"
+                            uppr++;
+                            break;
+                        case 5: // Recognized '{Utf8pfx4}{Utf8cont}',	Shortest string "\xF0\x80"
+                            uppr += 2;
+                            break;
+                        case 6: // Recognized '{Utf8pfx4}{Utf8cont}{2}',	Shortest string "\xF0\x80\x80"
+                            uppr += 3;
+                            break;
+                        case 7: // Recognized '{Utf8pfx4}{Utf8cont}{3}',	Shortest string "\xF0\x80\x80\x80"
+                            utfX += 3;
+                            break;
+                        case 8: // Recognized '{Utf8pfx3}{Utf8cont}',	Shortest string "\xE0\x80"
+                            uppr += 2;
+                            break;
+                        case 9: // Recognized '{Utf8pfx3}{Utf8cont}{2}',	Shortest string "\xE0\x80\x80"
+                            utfX += 2;
+                            break;
+                        case 10: // Recognized '{Utf8pfx2}{Utf8cont}',	Shortest string "\xC0\x80"
+                            utfX++;
+                            break;
+                        default:
+                            break;
+                    }
+#pragma warning restore 162
+                    #endregion
+                }
+            }
+        }
+    } // end class Guesser
+    
+#endif // !BYTEMODE
+#endregion
 #endif // !NOFILES
 
 // End of code copied from embedded resource

src/DataStreamSuite/Bts.ColumnExpressions/Bts.ColumnExpressions.csproj

@@ -1,7 +1,7 @@
 <Project Sdk="Microsoft.NET.Sdk">
 
   <PropertyGroup>
-    <TargetFramework>net8.0</TargetFramework>
+    <TargetFramework>net7.0</TargetFramework>
     <ImplicitUsings>enable</ImplicitUsings>
     <Nullable>enable</Nullable>
   </PropertyGroup>

src/DataStreamSuite/Bts.ColumnSortExpressions/Bts.ColumnSortExpressions.csproj

@@ -1,7 +1,7 @@
 <Project Sdk="Microsoft.NET.Sdk">
 
   <PropertyGroup>
-    <TargetFramework>net8.0</TargetFramework>
+    <TargetFramework>net7.0</TargetFramework>
     <ImplicitUsings>enable</ImplicitUsings>
     <Nullable>enable</Nullable>
   </PropertyGroup>

src/DataStreamSuite/ColumnMapExpressions/ColumnMapExpressions.csproj

@@ -1,7 +1,7 @@
 <Project Sdk="Microsoft.NET.Sdk">
 
   <PropertyGroup>
-    <TargetFramework>net8.0</TargetFramework>
+    <TargetFramework>net7.0</TargetFramework>
     <ImplicitUsings>enable</ImplicitUsings>
     <Nullable>enable</Nullable>
   </PropertyGroup>

src/DataStreamSuite/DataStream/BlueOrb/CountStream.cs

@@ -1,49 +0,0 @@
-using System;
-
-namespace BCG {
-   [DataStreamUserRespresentation("count")]
-   public class CountStream : ViewStream {
-      private DataPacket myLastFiredDataPacket;
-      private IBlueTableSchema mySchema = new BlueTableSchema("count",
-         new ColumnInfo("id", 0, typeof(int)),
-         new ColumnInfo("count", 1, typeof(int)));
-      private int myId = 0;
-      private ViewStream myVS;
-
-      public CountStream(ViewStream vs) {
-         myVS = vs;
-      }
-
-      public CountStream(string tableName)
-         : this(new TableStream(tableName)) { }
-
-      protected override void myOnFirstSubscriber() {
-         myVS.onChange += new ChangeDelegate(myHandleChanges);
-      }
-      protected override void myOnLastSubscriber() {
-         myVS.onChange -= new ChangeDelegate(myHandleChanges);
-      }
-
-
-      private void myHandleChanges(object newValue) {
-         myFireEvent(
-            myLastFiredDataPacket =
-               myLastFiredDataPacket == null ?
-                  new DataPacket(myGetRows(newValue, ++myId), myGetRows(newValue, myId),
-                  new BlueTable(mySchema))
-                  :
-                  new DataPacket(myGetRows(newValue, ++myId), myGetRows(newValue, myId),
-                  myLastFiredDataPacket.newRows)
-            );
-      }
-
-      private IBlueRowCollection myGetRows(object dp, int id) {
-         BlueTable retVal = new BlueTable(mySchema);
-         retVal.addRow
-            (new BlueRow(mySchema, id, (dp as DataPacket).newRows.Count));
-         return retVal;
-      }
-
-
-   }
-}

src/DataStreamSuite/DataStream/BlueOrb/DateTimeExpanderStream.cs

@@ -1,38 +0,0 @@
-namespace bts.BlueOrb {
-   using bts;
-
-   using BCG;
-
-   using System;
-   [DataStreamUserRespresentation("DateTimeExpanderStream")]
-   public class DateTimeExpanstionStream : ViewStream {
-      public DateTimeExpanstionStream(ViewStream parent_stream, string inp_col) {
-         Func<IBlueTableSchema, IBlueTableSchema> get_schema = (s) => {
-            IBlueTableSchema rv = s.add_column(System.TypeCode.DateTime, $"{inp_col}_date");
-            rv = rv.add_column(System.TypeCode.String, $"{inp_col}_day_of_week");
-            rv = rv.add_column(System.TypeCode.Int32, $"{inp_col}_day_of_month");
-            rv = rv.add_column(System.TypeCode.Int32, $"{inp_col}_month");
-            rv = rv.add_column(System.TypeCode.Int32, $"{inp_col}_hour");
-            rv = rv.add_column(System.TypeCode.Int32, $"{inp_col}_minute");
-            rv = rv.add_column(System.TypeCode.Int32, $"{inp_col}_seconds_of_day");
-            return rv;
-         };
-         ViewStream rs = new MapStream(parent_stream,
-         s => get_schema(s),
-         r => {
-            DateTime dt = r.get_datetime(inp_col);
-            IBlueTableSchema s = get_schema(r.schema);
-            MutableBlueRow rv = new MutableBlueRow(s, r);
-            rv.setValue($"{inp_col}_date", dt.Date);
-            rv.setValue($"{inp_col}_day_of_week", dt.DayOfWeek.ToString());
-            rv.setValue($"{inp_col}_day_of_month", dt.Day);
-            rv.setValue($"{inp_col}_month", dt.Date.Month);
-            rv.setValue($"{inp_col}_hour", dt.Hour);
-            rv.setValue($"{inp_col}_minute", dt.Minute);
-            rv.setValue($"{inp_col}_seconds_of_day", (dt - dt.Date).TotalSeconds);
-            return rv;
-         });
-         mySetRealSource(rs);
-      }
-   }
-}

src/DataStreamSuite/DataStream/BlueOrb/FilterStream.cs

@@ -1,158 +0,0 @@
-namespace bts.BlueOrb {
-   using BCG;
-   using System;
-   using System.Text.RegularExpressions;
-
-
-   public class RegExpFilter : IStreamFilter {
-      private Regex[] myRegExps;
-      private string[] myCols;
-
-      public RegExpFilter(string col, string regExp)
-         : this(new string[] { col }, new string[] { regExp }) { }
-
-      public RegExpFilter(string[] cols, string[] regExps) {
-         myCols = cols;
-         myRegExps = new Regex[regExps.Length];
-         for (int i = 0; i < regExps.Length; i++)
-            myRegExps[i] = new Regex(regExps[i]);
-      }
-
-      #region IStreamFilter Members
-      public bool filter(IBlueRow dr) {
-         bool rv = true;
-         for (int i = 0; i < myCols.Length; i++) {
-            rv &= myRegExps[i].IsMatch(dr[myCols[i]] is DBNull ? "" : dr[myCols[i]].ToString());
-            if (!rv)
-               return rv;
-         }
-         return rv;
-      }
-      #endregion
-   }
-
-   //[DataStreamUserReprsentation("FilterStream")]
-   public class FilterStream : ViewStream {
-      private BlueTable myLastFiredRows;
-      private IBlueRowCollection myLastReceivedRows;
-      private IStreamFilter myFilter;
-      private ViewStream myParentStream;
-      private string myFilterString;
-      private bool myUseNewFilter = false;// Configuration.get<DataStreamConfig>().UseNewFilter;
-
-      #region constructors
-      public delegate bool OnFilter(IBlueRow r);
-      private class OnFilterWrapper : IStreamFilter {
-         public OnFilterWrapper(OnFilter f) { this.f = f; }
-
-         public bool filter(IBlueRow dr) { return f(dr); }
-
-         private OnFilter f;
-      }
-      public FilterStream(ViewStream vs, OnFilter f)
-         : this(vs, new OnFilterWrapper(f)) { }
-
-
-      public FilterStream(ViewStream vs, IStreamFilter filter) {
-         {
-            myParentStream = vs;
-            myFilter = filter;
-            myFilterString = filter.ToString();
-            if (filter is IDynamicStreamFilter)
-               (filter as IDynamicStreamFilter).onFilterChange += new FilterChangeDelegate(myHandleFilterChange);
-         }
-      }
-
-      public FilterStream(ViewStream vs, string filterExpression)
-         : this(vs, new BooleanExpressionFilter(filterExpression)) {
-         myFilterString = filterExpression;
-      }
-
-      public FilterStream(ViewStream vs, string col, string regExp)
-         : this(vs, new RegExpFilter(col, regExp)) {
-         myFilterString = col + " , " + regExp;
-      }
-
-      public FilterStream(ViewStream vs)
-         : this(vs, new BinaryFilter(true)) {
-         myFilterString = true.ToString();
-      }
-      #endregion
-
-      #region overridden methods
-      protected override void myOnFirstSubscriber() { myParentStream.onChange += new ChangeDelegate(myHandleCallbacks); }
-
-      protected override void myOnLastSubscriber() {
-         myLastFiredRows = null;
-         myParentStream.onChange -= new ChangeDelegate(myHandleCallbacks);
-      }
-
-      protected override void myFireEvent(object o) {
-         DataPacket dp = o as DataPacket;
-         if (myLastValue == null || dp.addedRows.Count + dp.deletedRows.Count > 0)
-            base.myFireEvent(o);
-      }
-      #endregion
-
-      private void myHandleCallbacks(object newValue) {
-         DataPacket retVal;
-         DataPacket arrivedPacket = newValue as DataPacket;
-         myLastReceivedRows = arrivedPacket.newRows;
-         if (myLastFiredRows != null) {
-            retVal = new DataPacket(myLastFiredRows, new BlueTable(myLastFiredRows.schema),
-               new BlueTable(myLastFiredRows.schema));
-            foreach (IBlueRow r in arrivedPacket.deletedRows)
-               if (myLastFiredRows.containsID((int)r["id"])) {
-                  myLastFiredRows.deleteRow(r);
-                  (retVal.deletedRows as BlueTable).addRow(r);
-               }
-            foreach (IBlueRow r in arrivedPacket.addedRows)
-               if (myFilter.filter(r)) {
-                  myLastFiredRows.addRow(r);
-                  (retVal.addedRows as BlueTable).addRow(r);
-               }
-         } else {
-            myLastFiredRows = new BlueTable(arrivedPacket.schema);
-            retVal = new DataPacket(myLastFiredRows, myLastFiredRows, new BlueTable(myLastFiredRows.schema));
-            foreach (IBlueRow r in arrivedPacket.newRows)
-               if (myFilter.filter(r))
-                  myLastFiredRows.addRow(r);
-         }
-         myFireEvent(retVal);
-      }
-
-      public IStreamFilter filter {
-         set {
-            {
-               myFilter = value;
-               if (numberOfSubscribers > 0) {
-                  if (myLastReceivedRows != null) {
-                     BlueTable nRows = new BlueTable(myLastReceivedRows.schema);
-                     foreach (IBlueRow r in myLastReceivedRows)
-                        if (myFilter.filter(r))
-                           nRows.addRow(r);
-                     IBlueRowCollection dRows = new BlueTable(nRows.schema),
-                        aRows = new BlueTable(nRows.schema);
-                     BlueDataUtility.computeDifferences(nRows, myLastFiredRows, out dRows, out aRows);
-                     myLastFiredRows = nRows;
-                     myFireEvent(new DataPacket(nRows, aRows, dRows));
-                  }
-               }
-            }
-         }
-         get {            
-               return myFilter;
-         }
-      }
-
-      //public string filterString { set { filter = new BooleanExpressionFilter(value); } }
-
-      public override string ToString() {
-         return GetType().ToString() + " " + myFilterString;
-      }
-
-      private void myHandleFilterChange() {
-         filter = myFilter;
-      }
-   }
-}

src/DataStreamSuite/DataStream/BlueOrb/GroupByStream.cs

@@ -1,121 +0,0 @@
-using System;
-using System.Collections.Generic;
-using System.Diagnostics;
-using System.Linq;
-using System.Text.Json;
-
-using bts;
-//using bts.SilverPine;
-using BCG;
-
-namespace bts.BlueOrb {
-   [DataStreamUserRespresentation("GroupByStream")]
-   public class GroupByStream : ViewStream {
-
-      public interface INonIncGroupByEvaluator {
-         // grouped_rows
-         IBlueRowCollection evaluate(IDictionary<int, List<IBlueRow>> grouped_rows);
-         string[] get_group_by_cols();
-      }
-
-      private class DelNonIncGroupByEvaluator : INonIncGroupByEvaluator {
-         Func<string[]> _groupby_cols_func;
-         Func<List<IBlueRow>, List<IBlueRow>> _evaluate_func;
-         public DelNonIncGroupByEvaluator(Func<string[]> groupby_cols_func,
-                                          Func<List<IBlueRow>, List<IBlueRow>> evaluate_func) {
-            _groupby_cols_func = groupby_cols_func;
-            _evaluate_func = evaluate_func;
-         }
-         public IBlueRowCollection evaluate(IDictionary<int, List<IBlueRow>> grouped_rows) {
-            BlueTable rv = null;
-            foreach (var kv in grouped_rows) {
-               List<IBlueRow> g_rows = kv.Value;
-               List<IBlueRow> out_rows = _evaluate_func.Invoke(g_rows);
-               if (out_rows != null && out_rows.Count > 0) {
-                  rv = rv ?? new BlueTable(out_rows.ElementAt(0).schema);
-               }
-               rv.addRows(out_rows);
-            }
-            return rv;
-         }
-         public string[] get_group_by_cols() {
-            return _groupby_cols_func();
-         }
-
-      }
-      private ViewStream _main_stream;
-      private INonIncGroupByEvaluator _groupby_func;
-      private BlueRowHasher _row_hasher;
-      private string[] _groupby_cols;
-      private Stopwatch _timer = new();
-      private int _n_total_fires = 0, _n_expensive_fires = 0;
-
-      public GroupByStream(ViewStream main_stream, INonIncGroupByEvaluator groupby_func) {
-         _main_stream = main_stream;
-         _groupby_func = groupby_func;
-         _groupby_cols = _groupby_func.get_group_by_cols();
-         _row_hasher = new BlueRowHasher(_groupby_cols == null ? 0 : _groupby_cols.Length);
-      }
-
-      public GroupByStream(ViewStream main_stream, Func<string[]> groupby_cols_func,
-                                 Func<List<IBlueRow>, List<IBlueRow>> evaluate_func)
-                                 : this(main_stream, new DelNonIncGroupByEvaluator(groupby_cols_func, evaluate_func)) { }
-
-      public GroupByStream(ViewStream parent_str, string groupby_col, string standard_group_by_text)
-      : this(parent_str, groupby_col, standard_group_by_text, String.Empty) { }
-
-      // this is the main constructor that does everything
-      public GroupByStream(ViewStream parent_str, string groupby_col, string standard_group_by_ops, string optional_sort_col) {
-         MultiRowOpHelper std_group_by = MultiRowOpHelper.parse_ops(standard_group_by_ops);
-         Func<string[]> groupby_cols_func = () => {
-            return String.IsNullOrEmpty(groupby_col) ? null : new string[] { groupby_col };
-         };
-         Func<List<IBlueRow>, List<IBlueRow>> evaluate_func = (group_rows) => {
-            if (!String.IsNullOrEmpty(optional_sort_col))
-               group_rows.Sort((r1, r2) => BlueDataUtility.compare_rows(r2, r1, optional_sort_col));
-            // this si where the group by aggregation happens
-            IBlueRow rv = MultiRowOpHelper.group_by_compute(group_rows, std_group_by);
-            return new IBlueRow[] { rv }.ToList();
-         };
-         ViewStream rs = new GroupByStream(parent_str, groupby_cols_func, evaluate_func);
-         mySetRealSource(rs);
-      }
-      public GroupByStream(ViewStream parent_str, string standard_group_by_text) :
-      this(parent_str, String.Empty, standard_group_by_text) {
-      }
-
-
-      protected override void myOnFirstSubscriber() { _main_stream.onChange += _main_stream_onchange; }
-
-      protected override void myOnLastSubscriber() { _main_stream.onChange -= _main_stream_onchange; }
-
-      private void _main_stream_onchange(object nv) {
-         DataPacket dp = nv as DataPacket;
-         ///grouping together
-         _timer.Restart();
-         IDictionary<int, List<IBlueRow>> grouped_rows = (_groupby_cols == null || _groupby_cols.Length == 0) ?
-         new Dictionary<int, List<IBlueRow>>() { { 0, dp.newRows.to_list() } } :
-         BlueDataUtility.group_together(dp.newRows, _groupby_cols, ref _row_hasher);
-         _timer.Stop();
-         var groupElapsedMs = _timer.ElapsedMilliseconds;
-         if (groupElapsedMs >= 50) {
-            Log.DataFlow.info($"Time to group {dp.newRows.Count} rows into {grouped_rows.Count} groups: {_timer.ElapsedMilliseconds}ms");
-         }
-         // Log.log(bts.Log.Level.Info, bts.log.Subsystem.gen, String.Format("Time to group {0} rows into {1} groups = {2}", dp.newRows.Count, grouped_rows.Count, _timer.lastIntervalTime));
-         /// reducing
-         _timer.Restart();
-         IBlueRowCollection total_state = _groupby_func.evaluate(grouped_rows);
-         _timer.Stop();
-         var applyElapsedMs = _timer.ElapsedMilliseconds;
-         _n_total_fires++;
-         if (applyElapsedMs >= 50) {
-            _n_expensive_fires++;
-            Log.DataFlow.info(
-               $"Time to evaluate {grouped_rows.Count} groups {_timer.ElapsedMilliseconds}ms, ratio {_n_expensive_fires}/{_n_total_fires}");
-         }
-         // TODO,
-         if (total_state != null)
-            myFireEvent(new DataPacket(total_state, total_state, myLastValue == null ? new BlueTable(total_state.schema) : (myLastValue as DataPacket).newRows));
-      }
-   }
-}

src/DataStreamSuite/DataStream/BlueOrb/Join/INonIncJoinInlineMapper.cs

@@ -1,17 +0,0 @@
-using System;
-using System.Collections.Generic;
-using System.Linq;
-using System.Text;
-using System.Threading.Tasks;
-using BCG;
-
-namespace bts.BlueOrb {   
-   public interface INonIncJoinInlineMapper
-   {      
-      IBlueTableSchema get_output_schema(IBlueTableSchema schema);
-
-      IBlueRowCollection map_rows(IBlueRowCollection rows);
-      
-      event MapperChangeDelegate onMapperChange;
-   }
-}

src/DataStreamSuite/DataStream/BlueOrb/Join/NonIncJoinBookkeeper.cs

@@ -1,152 +0,0 @@
-using bts;
-using bts.collections;
-
-using mbts;
-
-using System;
-using System.Collections.Generic;
-using System.Diagnostics;
-using System.Linq;
-using System.Security.Cryptography;
-using BCG;
-
-namespace bts.BlueOrb {   
-   internal class NonIncJoinBookkeeper : IDisposable {
-      public Action on_new_data;
-      private readonly NonIncJoinSpec _join_spec;
-      private readonly int _join_position;
-      private readonly int _start_index_in_output;
-      private readonly BlueRowHasher _row_hasher;
-      private readonly bool _is_left_most;
-      private readonly bool _is_tag_along;
-      private int _n_total_fires = 0, _n_expensive_fires = 0;
-
-      private IBlueTableSchema _schema = null;
-      private IDictionary<int, IBlueRow> _row_map = new Dictionary<int, IBlueRow>();
-      private List<Pair<int, IBlueRow>> _row_list = new List<Pair<int, IBlueRow>>();
-      private IBlueRow _solitary_row = null;
-
-      private int[] _join_col_indices = null,
-                     _data_col_indices = null;
-
-      public NonIncJoinBookkeeper(NonIncJoinSpec the_join_spec, int join_position, int start_index_in_output, BlueRowHasher row_hasher) {
-         _join_spec = the_join_spec;
-         _join_position = join_position;
-         _start_index_in_output = start_index_in_output;
-         _row_hasher = row_hasher;
-         _is_left_most = _join_position == 0;
-         _is_tag_along = _join_spec.join_col_names.Length == 0;
-      }
-
-      private bool _filled_missing_data = false;
-      private void _fill_missing_data(DataPacket dp) {
-         if (_filled_missing_data)
-            return;
-         Func<string[], bool> null_or_empty = (str_arr) => {
-            if (str_arr == null || str_arr.Length == 0)
-               return true;
-            foreach (string s in str_arr) {
-               if (!String.IsNullOrEmpty(s)) {
-                  return false;
-               }
-            }
-            return true;
-         };
-         Func<string[], string[]> remove_bad_cols = (col_names) => {
-            HashSet<string> schema_col_map = new HashSet<string>(dp.schema.columnNames);
-            List<string> cols_list = new List<string>(col_names);
-            cols_list.RemoveWhere(c => !schema_col_map.Contains(c));
-            return cols_list.ToArray();
-         };
-         if (null_or_empty(_join_spec.data_col_names)) {
-            _join_spec.data_col_names = dp.schema.columnNames;
-         }
-         _join_spec.data_col_names = remove_bad_cols(_join_spec.data_col_names);
-         if (null_or_empty(_join_spec.data_col_names)) {
-            _join_spec.data_col_names = dp.schema.columnNames;
-         }
-         if (null_or_empty(_join_spec.projected_col_names)) {
-            _join_spec.projected_col_names = _join_spec.data_col_names;
-         }
-      }
-
-      public void init() {
-         _join_spec.stream.onChange += _on_stream_cb;
-      }
-
-      public void _on_stream_cb(object dp_obj) {
-         //Log.Gen.info($"NonIncJoinBookkeeper._on_stream_cb dp_obj: {dp_obj}");
-         var timer = Stopwatch.StartNew();
-         DataPacket dp = (DataPacket)dp_obj;
-         _fill_missing_data(dp);
-         if (_schema == null) {
-            _schema = dp.schema;
-            _join_col_indices = BlueDataUtility.get_column_indices(_schema, _join_spec.join_col_names);
-            _data_col_indices = BlueDataUtility.get_column_indices(_schema, _join_spec.data_col_names);
-         }
-
-         if (_is_left_most) {
-            _row_list.Clear();
-            foreach (IBlueRow r in dp.newRows)
-               _row_list.Add(new Pair<int, IBlueRow>(_row_hasher.get_hash(r, _join_col_indices), r));
-         } else {
-            if (_is_tag_along)
-               _solitary_row = BlueDataUtility.extractRow(dp.newRows, 0);
-            else {
-               _row_map.Clear();
-               foreach (IBlueRow r in dp.newRows)
-                  _row_map[_row_hasher.get_hash(r, _join_col_indices)] = r;
-            }
-         }
-         timer.Stop();
-         _n_total_fires++;
-         if (timer.ElapsedMilliseconds >= 50) {
-            _n_expensive_fires++;
-            Log.Gen.info($"NonIncJoinBookkeepingData._on_stream_cb {dp.newRows.Count} rows {_join_col_indices.Length} "
-               + $"key cols {_join_position} join pos time {timer.Elapsed} ratio {_n_expensive_fires}/{_n_total_fires}");
-         }
-         if (on_new_data != null)
-            on_new_data();
-      }
-
-      public bool is_ready { get { return _schema != null; } }
-
-      public void constribute_cols_to_schema(List<ColumnInfo> cis) {
-         for (int i = 0; i < _join_spec.data_col_names.Length; i++)
-            cis.Add(new ColumnInfo(_join_spec.projected_col_names[i], cis.Count, _schema[_join_spec.data_col_names[i]].colType));
-      }
-
-      public List<Pair<int, IBlueRow>> get_list() { return _row_list; }
-
-      public void copy_master_row_to_output_row(int row_index, object[] output_row) {
-         IBlueRow this_row = _row_list[row_index].second;
-         BlueDataUtility.copy_to_row(this_row, _data_col_indices, output_row, _start_index_in_output);
-      }
-
-      public void copy_matching_row_to_output_row(int row_hash, object[] output_row) {
-         if (_is_tag_along) {
-            if (_solitary_row != null)
-               BlueDataUtility.copy_to_row(_solitary_row, _data_col_indices, output_row, _start_index_in_output);
-            else
-               copy_dbnull_to_output_row(output_row);
-         } else {
-            IBlueRow this_row = null;
-            if (_row_map.TryGetValue(row_hash, out this_row))
-               BlueDataUtility.copy_to_row(this_row, _data_col_indices, output_row, _start_index_in_output);
-            else
-               copy_dbnull_to_output_row(output_row);
-         }
-      }
-
-      private void copy_dbnull_to_output_row(object[] output_row) {
-         for (int i = 0; i < _data_col_indices.Length; i++)
-            output_row[i + _start_index_in_output] = DBNull.Value;
-      }
-
-      #region IDisposable
-
-      public void Dispose() { _join_spec.stream.onChange -= _on_stream_cb; }
-
-      #endregion
-   }
-}

src/DataStreamSuite/DataStream/BlueOrb/Join/NonIncJoinSpec.cs

@@ -1,67 +0,0 @@
-using System;
-using System.Collections.Generic;
-using System.Linq;
-using System.Text;
-using System.Threading.Tasks;
-
-using BCG;
-
-using mbts;
-
-namespace bts.BlueOrb {   
-   // this a collection of spec, theres no joining logic in here
-   public class NonIncJoinSpec {
-      public ViewStream stream;
-      // join col names are the names on which the streams are joined (for equality),
-      // data col names are the names in the original stream
-      // projected col names are their new names, ie if its x:l_x, then x is in the 
-      // data_col_names and l_x in projected cols
-      public string[] join_col_names, data_col_names, projected_col_names;
-
-      public NonIncJoinSpec() { }
-
-      public NonIncJoinSpec(ViewStream the_stream, string[] the_join_col_names, string[] the_data_col_names)
-         : this(the_stream, the_join_col_names, the_data_col_names, the_data_col_names) { }
-
-      public NonIncJoinSpec(ViewStream the_stream, string[] the_join_col_names, string[] the_data_col_names, string[] the_projected_col_names) {
-         stream = the_stream;
-         join_col_names = the_join_col_names;
-         data_col_names = the_data_col_names;
-         projected_col_names = the_projected_col_names;
-      }
-
-      public static NonIncJoinSpec[] from_triplet(params Triplet<ViewStream, string, string>[] join_specs) {
-         List<NonIncJoinSpec> rv = new List<NonIncJoinSpec>();
-         foreach (Triplet<ViewStream, string, string> r in join_specs) {
-            string[] keys = String.IsNullOrEmpty(r.second) ? new string[] { } : r.second.Split(' ');
-            string[] visible_cols = r.third.Split(' ');
-            List<string> data_col_names = new List<string>(), mapped_col_names = new List<string>();
-            foreach (string s in visible_cols) {
-               string[] toks = s.Split(':');
-               if (toks.Length == 1) {
-                  data_col_names.Add(toks[0]);
-                  mapped_col_names.Add(toks[0]);
-               } else if (toks.Length == 2) {
-                  data_col_names.Add(toks[0]);
-                  mapped_col_names.Add(toks[1]);
-               }
-            }
-            rv.Add(new NonIncJoinSpec(r.first, keys, data_col_names.ToArray(), mapped_col_names.ToArray()));
-         }
-         return rv.ToArray();
-      }
-
-      public override string ToString() {
-         Func<string[], string> arr_txt = (a) => {
-            string rv = "";
-            foreach (string s in a) {
-               rv += "{s} ";
-            }
-            rv = rv.Trim();
-            return rv;
-         };
-         string rv = $"stream: {stream} join_col_names: {arr_txt(join_col_names)} data_col_names: {arr_txt(data_col_names)} projected_col_names: {arr_txt(projected_col_names)}";
-         return rv;
-      }
-   }
-}

src/DataStreamSuite/DataStream/BlueOrb/Join/NonIncJoinStream.cs

@@ -1,165 +0,0 @@
-using bts;
-
-using mbts;
-
-using System;
-using System.Collections.Generic;
-using System.Diagnostics;
-using BCG;
-
-namespace bts.BlueOrb {   
-   [DataStreamUserRespresentation("JoinStream")]
-   public class NonIncJoinStream : ViewStream, BCG.Utilities.General.IWorker {
-      private readonly NonIncJoinSpec[] _join_specs;
-      private readonly INonIncJoinInlineMapper _output_mapper;
-      private readonly BlueRowHasher _row_hasher;
-      private readonly TimeSpan _throttle_period;
-      private readonly ViewStreamDatabase _vsd;
-
-      private List<NonIncJoinBookkeeper> _book_keeping_data = new List<NonIncJoinBookkeeper>();
-      private IBlueTableSchema _output_schema = null;
-      private DateTime _last_fire_time = DateTime.MinValue;
-      private bool _is_dirty = false;
-
-      public NonIncJoinStream(ViewStream left, string left_join_cols, string left_proj,
-                              ViewStream right, string right_join_cols, string right_proj)
-         : this(new Triplet<ViewStream, string, string>[]
-               {
-                  new Triplet<ViewStream, string, string> (left, left_join_cols, left_proj),
-                  new Triplet<ViewStream, string, string> (right, right_join_cols, right_proj)
-               }) { }
-
-      public NonIncJoinStream(params Triplet<ViewStream, string, string>[] join_specs)
-         : this(NonIncJoinSpec.from_triplet(join_specs), null, 0) { }
-
-      public NonIncJoinStream(ViewStreamDatabase vsd, INonIncJoinInlineMapper output_mapper, int throttles_ms,
-                              params Triplet<ViewStream, string, string>[] join_specs)
-         : this(vsd, NonIncJoinSpec.from_triplet(join_specs), output_mapper, throttles_ms) { 
-      }
-
-      public NonIncJoinStream(NonIncJoinSpec[] join_specs, INonIncJoinInlineMapper output_mapper, int throttles_ms = 0)
-         : this(ViewStreamDatabase.instance, join_specs, output_mapper, throttles_ms) { }
-
-      public NonIncJoinStream(ViewStreamDatabase vsd, NonIncJoinSpec[] join_specs, INonIncJoinInlineMapper output_mapper, int throttles_ms = 0) {
-         _join_specs = join_specs;
-         _output_mapper = output_mapper;
-         _row_hasher = new BlueRowHasher(_join_specs[0].join_col_names.Length);
-         _throttle_period = throttles_ms == 0 ? TimeSpan.Zero : new TimeSpan(0, 0, 0, 0, throttles_ms);
-         _vsd = vsd;
-      }
-
-      protected override void myOnFirstSubscriber() {
-         int data_col_no = 1;
-         for (int jp = 0; jp < _join_specs.Length; jp++) {
-            NonIncJoinBookkeeper s = new NonIncJoinBookkeeper(_join_specs[jp], jp, data_col_no, _row_hasher);
-            //Log.Gen.info($"NonIncJoinStream.myOnFirstSubscriber jp: {jp} spec: {_join_specs[jp]}");
-            data_col_no += _join_specs[jp].data_col_names.Length;
-            s.on_new_data += _stream_callback;
-            _book_keeping_data.Add(s);
-         }
-         if (_output_mapper != null)
-            _output_mapper.onMapperChange += _on_mapper_change;
-         _vsd.addWorker(this);
-         foreach (var bkd in _book_keeping_data)
-            bkd.init();
-      }
-
-      protected override void myOnLastSubscriber() {
-         foreach (NonIncJoinBookkeeper d in _book_keeping_data)
-            d.Dispose();
-         if (_output_mapper != null)
-            _output_mapper.onMapperChange -= _on_mapper_change;
-         _vsd.removeWorker(this);
-      }
-
-      private void _stream_callback() {
-         //Log.Gen.info("NonIncJoinStream._stream_callback received callback");
-         _is_dirty = true;
-         doWork();
-      }
-
-      private void _on_mapper_change() {
-         _is_dirty = true;
-         doWork();
-      }
-
-      private bool _is_ready() {
-         bool rv = true;
-         foreach (NonIncJoinBookkeeper d in _book_keeping_data)
-            rv &= d.is_ready;
-         return rv;
-      }
-
-      private void _construct_schema() {
-         if (_output_schema == null) {
-            List<ColumnInfo> cis = new List<ColumnInfo>(new ColumnInfo[] { new ColumnInfo("id", 0, typeof(int)) });
-            foreach (NonIncJoinBookkeeper d in _book_keeping_data)
-               d.constribute_cols_to_schema(cis);
-            _output_schema = new BlueTableSchema("NonIncJoinStream", cis.ToArray());
-         }
-      }
-
-      private int _n_total_fires = 0, _n_expensive_fires = 0;
-      private void _form_and_fire() {
-         //Log.Gen.info($"NonIncJoinStream._form_and_fire _is_ready(): {_is_ready()}");
-         if (_is_ready()) {
-            _construct_schema();
-            var timer = Stopwatch.StartNew();
-            IList<Pair<int, IBlueRow>> left_row_list = _book_keeping_data[0].get_list();
-            List<object[]> output_rows = new List<object[]>();
-
-            for (int i = 0; i < left_row_list.Count; i++) {
-               object[] or = new object[_output_schema.columnInfo.Length];
-               or[0] = i;
-               output_rows.Add(or);
-            }
-
-            for (int ji = 0; ji < _book_keeping_data.Count; ji++) {
-               NonIncJoinBookkeeper bk_data = _book_keeping_data[ji];
-               for (int i = 0; i < left_row_list.Count; i++) {
-                  if (ji == 0)
-                     bk_data.copy_master_row_to_output_row(i, output_rows[i]);
-                  else
-                     bk_data.copy_matching_row_to_output_row(left_row_list[i].first, output_rows[i]);
-               }
-            }
-            IBlueRowCollection dp_output_rows = _apply_map(new SequentialIDBlueTable(_output_schema, output_rows));
-            timer.Stop();
-            _n_total_fires++;
-            //if (timer.ElapsedMilliseconds >= 250) {
-            _n_expensive_fires++;
-            //string log_msg = $"Join and map time for {_book_keeping_data.Count} "
-            //   + $"tables {timer.Elapsed.TotalMilliseconds} msecs, ratio {_n_expensive_fires}/{_n_total_fires}";
-            //Log.Gen.info(log_msg);
-            //}
-            var dp_out = BlueDataUtility.build_dp_inc(dp_output_rows, myLastValue as DataPacket, true);
-            myFireEvent(dp_out);
-            //myFireEvent(new DataPacket(dp_output_rows, dp_output_rows, myLastValue == null ? new BlueTable(_output_schema) : (myLastValue as DataPacket).newRows));
-         }
-      }
-
-      private IBlueRowCollection _apply_map(IBlueRowCollection unmapped_rows) {
-         if (_output_mapper != null) {
-            IBlueRowCollection output_rows;
-            var timer = Stopwatch.StartNew();
-            output_rows = _output_mapper.map_rows(unmapped_rows);
-            timer.Stop();
-            if (timer.ElapsedMilliseconds >= 50) {
-               Log.Gen.info($"Time to map {unmapped_rows.Count} rows into {output_rows.Count} rows {timer.Elapsed}");
-            }
-            return output_rows;
-         } else {
-            return unmapped_rows;
-         }
-      }
-
-      public void doWork() {
-         //Log.Gen.info($"NonIncJoinStream.doWork _last_fire_time: {_last_fire_time} _is_dirty: {_is_dirty}");
-         if (DateTime.Now - _last_fire_time >= _throttle_period && _is_dirty) {
-            _is_dirty = false;
-            _last_fire_time = DateTime.Now;
-            _form_and_fire();
-         }
-      }
-   }
-}

src/DataStreamSuite/DataStream/BlueOrb/JsonSaverStream.cs

@@ -1,28 +0,0 @@
-//namespace bts.Blueorb {
-//   using BCG;
-
-//   public class JsonSaverStream : ViewStream {
-//      private ViewStream _stream;
-//      private string _file_name;
-//      public JsonSaverStream(ViewStream vs, string file_name) {
-//         _stream = vs;
-//         _file_name = file_name;
-//      }
-
-//      protected override void myOnFirstSubscriber() {
-//         _stream.onChange += _on_data;
-//      }
-
-//      private void _on_data(object dp_obj) {
-//         if (dp_obj == null) {
-//            return;
-//         }
-//         DataPacket dp = dp_obj as DataPacket;
-//         dp.newRows.write_to_json_file(dp.newRows, _file_name);
-//      }
-
-//      protected override void myOnLastSubscriber() {
-//         _stream.onChange -= _on_data;
-//      }
-//   }
-//}

src/DataStreamSuite/DataStream/BlueOrb/LiteralStream.cs

@@ -1,91 +0,0 @@
-using System;
-using System.Collections;
-using System.Collections.Generic;
-
-using Amazon.Runtime.Internal.Util;
-
-using BCG;
-using BCG.Utilities;
-
-using bts;
-
-namespace bts.BlueOrb{
-   public class LiteralStream : ViewStream {
-
-      public static List<Tuple<ColumnInfo, object>> parse_columns(string key_value_pairs) {
-         List<Tuple<ColumnInfo, object>> rv = new List<Tuple<ColumnInfo, object>>();
-         string[] pairs = key_value_pairs.Split(' ');
-         foreach (string p in pairs) {
-            string[] kvp = p.Split(':');
-            string col_name = kvp[0];
-            string val = kvp[1];
-            var tup = BlueDataUtility.guess_type(val);
-            var ci = new ColumnInfo(col_name, rv.Count, tup.Item1);
-            rv.Add(Tuple.Create(ci, tup.Item2));
-         }
-         return rv;
-      }
-
-      //literal("table1", "x:123, y:1.001, z:abc;x:123, y:1.001, z:abc")
-      // col_name: col_value ... ; col_name: col_value
-      public LiteralStream(string table_name, string rows) {
-         Func<string, int, IBlueRow> get_row = (json_row, id) => {
-            Dictionary<string, object> map = new Dictionary<string, object>();
-            string[] toks = json_row.Split(',', StringSplitOptions.TrimEntries);
-            //Logger.assert(col_vals.Length == 2);
-            foreach (string t in toks) {
-               var col_vals = t.Split(':', StringSplitOptions.TrimEntries);
-               string col = col_vals[0];
-               string val = col_vals[1];
-               map.Add(col, val);
-            }
-            map["id"] = id;
-            return new BlueRow(map);
-         };
-         string[] row_arr = rows.Split(';', StringSplitOptions.TrimEntries);
-         BlueTable table = null;
-         foreach (string json_string in row_arr) {
-            IBlueRow r = //BlueDataUtility.convert_json_to_row(json_string, table == null ? 0 : table.Count);
-            get_row(json_string, table == null ? 0 : table.Count);
-            table = table ?? new BlueTable(r.schema);
-            table.addRow(r);
-         }
-         ViewStream str = new BlueTableStream(table);
-         str = new SchemaGuesserStream(str);
-         mySetRealSource(str);
-      }
-
-      public LiteralStream(string tableName, string schema, string value) {
-         string[] columns = schema.Split(' ');
-
-
-         ColumnInfo[] ci = new ColumnInfo[columns.Length + 1];
-         ci[0] = new ColumnInfo("id", 0, typeof(int));
-         int n = 1;
-         foreach (string c in columns) {
-            string[] columnAndType = c.Split('/');
-            if (columnAndType.Length == 1)
-               ci[n] = new ColumnInfo(c, n, typeof(double));
-            else
-               ci[n] = new ColumnInfo(columnAndType[0], n, Type.GetType("System." + columnAndType[1], true));
-            n++;
-         }
-
-
-         BlueTable t = new BlueTable(new BlueTableSchema(tableName, ci));
-         string[] rows = value.Split(';');
-         n = 0;
-         foreach (string r in rows)
-            try {
-               if (r == string.Empty)
-                  continue;
-               t.addRow(
-                  BlueDataUtility.createRowFromCSVFileRow(r, t.schema, ++n));
-            } catch (Exception e) {
-               throw (new Exception("Error occured at line: " + n.ToString() + " " + e.Message));
-            }
-
-         mySetRealSource(new BlueRowCollectionStaticStream(t));
-      }
-   }
-}

src/DataStreamSuite/DataStream/BlueOrb/MillisecondStream.cs

@@ -1,89 +0,0 @@
-using System;
-
-using BCG.Utilities.General;
-using BCG;
-using bts;
-
-namespace bts.BlueOrb {
-   [DataStreamUserRespresentation("MillisecondStream")]
-   public class MillisecondStream : ViewStream, IWorker {
-      private DateTime _last_fire_time = DateTime.MinValue;
-      private readonly TimeSpan _period;
-      private readonly int _size;
-      private readonly ViewStreamDatabase _vsd;
-      private IBlueTableSchema _schema;
-      private bts.CircularBufferTable _table;
-
-      public MillisecondStream(int interval_msecs, int size)
-      : this(ViewStreamDatabase.instance, interval_msecs, size) {
-      }
-
-      public MillisecondStream(ViewStreamDatabase vsd, int interval_msecs, int size) {
-         _vsd = vsd;
-         _period = TimeSpan.FromMilliseconds(interval_msecs);
-         _size = size;
-         _schema = BlueDataUtility.form_schema("Milliseconds",
-               Tuple.Create("id", TypeCode.Int32),
-               Tuple.Create("time", TypeCode.DateTime),
-               Tuple.Create("display_time", TypeCode.String));
-         _table = new CircularBufferTable(_schema, _size);
-      }
-
-      protected override void myOnFirstSubscriber() {
-         _vsd.addWorker(this);
-      }
-
-      protected override void myOnLastSubscriber() {
-         _vsd.removeWorker(this);
-      }
-
-      private int _id = 0;
-      public void doWork() {
-         if (DateTime.Now - _last_fire_time >= _period) {
-            _last_fire_time = DateTime.Now;
-            IBlueRow nr = new BlueRow(_schema, _id++, _last_fire_time,
-            _last_fire_time.ToString("yyyy-MM-dd HH:mm:ss.fff"));
-            _table.add_row(nr);
-            var dp = BlueDataUtility.build_dp_inc(_table, myLastValue as DataPacket);
-            myFireEvent(dp);
-         }
-      }
-   }
-   public class MSecondStream : TimeUnitStream {
-      public MSecondStream(int intervalInms, int size)
-         : base(DateTime.Now, new TimeSpan(0, 0, 0, 0, intervalInms), size, ViewStreamDatabase.instance) { }
-
-      protected override DateTime myRoundTime(DateTime dt) { return dt; }
-   }
-
-   public class DisplayableMillisecondStream : ViewStream, IStreamMapper {
-      private IBlueTableSchema mySchema = new BlueTableSchema("ms",
-                                                               new ColumnInfo("id", 0, typeof(int)),
-                                                               new ColumnInfo("time", 1, typeof(string)));
-      public DisplayableMillisecondStream(int intervalInMs, int sz) {
-         mySetRealSource(new MapStream(new MSecondStream(intervalInMs, sz), this));
-      }
-      #region IStreamMapper Members
-
-      public IBlueTableSchema getOutputSchema(IBlueTableSchema inputSchema) {
-         return mySchema;
-      }
-
-      #endregion
-
-      #region IDynamicStreamMapper Members
-
-      public IBlueRow map(IBlueRow row) {
-         DateTime time = row.get_datetime("time");
-         var time_txt = time.ToString("yyyy-MM-dd HH:mm:ss.fff");
-         return new BlueRow(mySchema, row["id"], time_txt);
-      }
-
-      #endregion
-
-      private string myToString(DateTime dt) {
-         string sep = ":";
-         return dt.Hour + sep + dt.Minute + sep + dt.Second + sep + dt.Millisecond;
-      }
-   }
-}

src/DataStreamSuite/DataStream/BlueOrb/MultiRowOpHelper.cs

@@ -1,127 +0,0 @@
-namespace bts.BlueOrb {
-   using BCG;
-
-   using System;
-   using System.Collections.Generic;
-   using System.Linq;
-   //using MathNet.Numerics.Statistics;
-
-
-   public enum MultiRowOp {
-      Mean,
-      Median,
-      Mode,
-      Range,
-      StdDev,
-      Max,
-      Min,
-      First,
-      Last,
-      Sum,
-      EdgeDiff,
-      EdgeDiffPercentage,
-      //IsEdgeDiff,
-   }
-   public class MultiRowOpHelper {
-      public readonly List<Tuple<string, MultiRowOp>> operations;
-
-      public static MultiRowOpHelper parse_ops(string text) {
-         // text is inp_col: op, inp_col: op, inp_col:op
-         List<Tuple<string, MultiRowOp>> rv = new List<Tuple<string, MultiRowOp>>();
-         string[] calcs = text.Split(',');
-         foreach (string c in calcs) {
-            string[] toks = c.Split(':');
-            string inp_col = toks[0].Trim();
-            string op_txt = toks[1].Trim();
-            MultiRowOp op = Enum.Parse<MultiRowOp>(op_txt, true);
-            rv.Add(Tuple.Create(inp_col, op));
-         }
-         return new MultiRowOpHelper(rv);
-      }
-
-      public MultiRowOpHelper(List<Tuple<string, MultiRowOp>> operations_) {
-         operations = operations_;
-      }
-
-      // aggregates multiple rows into one - ie does group_by compute
-      public static IBlueRow group_by_compute(List<IBlueRow> rows, MultiRowOpHelper parameters) {
-         if (rows.Count() == 0) {
-            return null;
-         }
-
-         IBlueRow ir = rows.ElementAt(0);
-         IBlueTableSchema in_schema = ir.schema;
-         IBlueTableSchema out_schema = in_schema;
-         IDictionary<string, Tuple<string, MultiRowOp>> out_col_map =
-            new Dictionary<string, Tuple<string, MultiRowOp>>();
-         foreach (var t in parameters.operations) {
-            string out_col_name = $"{t.Item1}_{t.Item2}".ToLower();
-            out_schema = out_schema.add_column(TypeCode.Double, out_col_name);
-            out_col_map[out_col_name] = t;
-         }
-         MutableBlueRow or = new MutableBlueRow(out_schema, ir);
-         foreach (var kv in out_col_map) {
-            string out_col_name = kv.Key;
-            string in_col_name = kv.Value.Item1;
-            MultiRowOp op = kv.Value.Item2;
-            IEnumerable<double> col_vals = BlueDataUtility.get_colvalues<double>(rows, in_col_name);
-            double comp_res = compute_op(col_vals, op);
-            or.setValue(out_col_name, comp_res);
-         }
-         return or;
-      }
-
-      // rolling_rows are the rolling window of rows. the 0th row is the one thats gettin
-      // calculated. rows 1 - n-1 are its window rows. the schema is based on the operations.
-
-      public IBlueRow rolling_calc(List<IBlueRow> rolling_rows) {
-         if (rolling_rows.Count() == 0) {
-            return null;
-         }
-         IBlueRow ir = rolling_rows.ElementAt(0);
-         IBlueTableSchema in_schema = ir.schema;
-         IBlueTableSchema out_schema = in_schema;
-         IDictionary<string, Tuple<string, MultiRowOp>> out_col_map =
-            new Dictionary<string, Tuple<string, MultiRowOp>>();
-         foreach (var t in operations) {
-            string out_col_name = $"{t.Item1}_{t.Item2}".ToLower();
-            out_schema = out_schema.add_column(TypeCode.Double, out_col_name);
-            out_col_map[out_col_name] = t;
-         }
-         MutableBlueRow or = new MutableBlueRow(out_schema, ir);
-         foreach (var kv in out_col_map) {
-            string out_col_name = kv.Key;
-            string in_col_name = kv.Value.Item1;
-            MultiRowOp op = kv.Value.Item2;
-            IEnumerable<double> col_vals = BlueDataUtility.get_colvalues<double>(rolling_rows, in_col_name);
-            double comp_res = compute_op(col_vals, op);
-            or.setValue(out_col_name, comp_res);
-         }
-         return or;
-      }
-
-      public static double compute_op(IEnumerable<double> col_vals, MultiRowOp op) {
-         switch (op) {
-            case MultiRowOp.Max:
-               return col_vals.Max<double>();
-            case MultiRowOp.Min:
-               return col_vals.Min<double>();
-            case MultiRowOp.Mean:
-               return col_vals.Sum() / col_vals.Count();
-            case MultiRowOp.Sum:
-               return col_vals.Sum();
-            case MultiRowOp.First:
-               return col_vals.First();
-            case MultiRowOp.Last:
-               return col_vals.Last();
-            case MultiRowOp.StdDev:
-               return MathNet.Numerics.Statistics.Statistics.StandardDeviation(col_vals);
-            default:
-               // TODO
-               Log.assert(false);
-               return Double.NaN;
-         }
-      }
-
-   }
-}

src/DataStreamSuite/DataStream/BlueOrb/ProjectStream.cs

@@ -1,207 +0,0 @@
-namespace bts.BlueOrb {
-   using BCG;
-
-   using ProtoBuf;
-
-   using System;
-   using System.Collections;
-   using System.Collections.Generic;
-   using System.Linq;
-
-   #region mappings
-   internal class ProjectData {
-      // schema from the base stream
-      public readonly IBlueTableSchema in_schema;
-      // this is the projected schema
-      public readonly IBlueTableSchema out_schema;
-      // projected cols to the input cols, for cols who's names are changed
-      public readonly IDictionary<string, string> mapped_to_orig_col_map = new Dictionary<string, string>();
-      public ProjectData(IBlueTableSchema in_schema_, IBlueTableSchema out_schema_,
-                  IDictionary<string, string> mapped_to_orig_col_map_) {
-         in_schema = in_schema_;
-         out_schema = out_schema_;
-         mapped_to_orig_col_map = mapped_to_orig_col_map_;
-      }
-   }
-   #endregion
-
-
-#region Row with mapipings build in
-   internal class NewProjectRow : IBlueRow {
-      #region IBlueRow members
-      private ProjectData _project_data;
-      private IBlueRow _row;
-      public NewProjectRow(ProjectData project_data, IBlueRow row) {
-         _project_data = project_data;
-         _row = row;
-      }
-      public IBlueTableSchema schema { get { return _project_data.out_schema; } }
-      #endregion
-
-      #region ICollection members
-      public int Count { get { return schema.columnNames.Length; } }
-
-      public bool IsSynchronized { get { return false; } }
-
-      public object SyncRoot { get { return this; } }
-
-      public void CopyTo(Array array, int index) {
-         int i = index;
-         foreach (object o in this as IEnumerable)
-            array.SetValue(o, i++);
-      }
-      #endregion
-
-      #region IDictionary members
-      public bool IsFixedSize { get { return true; } }
-
-      public bool IsReadOnly { get { return true; } }
-
-      public object this[object key] {
-         get {
-            if (key is string)
-               return this[(string)key];
-            return this[(int)key];
-         }
-         set { throw new NotSupportedException("The IBlueRow is readonly"); }
-      }
-
-      public object this[int index] {
-         get {
-            return this[schema.columnInfo[index].colName];
-         }
-      }
-
-      public object this[string col_name] {
-         get {
-            string mapped_to_col = null;
-            if (_project_data.mapped_to_orig_col_map.TryGetValue(col_name, out mapped_to_col)) {
-               return _row[mapped_to_col];
-            }
-            return _row[col_name];
-         }
-      }
-
-      public ICollection Keys { get { return schema.columnNames; } }
-
-      public ICollection Values {
-         get {
-            object[] retVal = new object[schema.columnNames.Length];
-            CopyTo(retVal, 0);
-            return retVal;
-         }
-      }
-
-      public void Add(object key, object value) { throw new NotSupportedException("The IBlueRow is readonly"); }
-
-      public void Clear() { throw new NotSupportedException("The IBlueRow is readonly"); }
-
-      public bool Contains(object key) { return schema[(string)key] != null; }
-
-      /// <summary>
-      /// Note, the default enumerator is the IDictionary's enumerator, so if user needs to
-      /// iterate through the values, typecast this into an IEnumerable
-      /// </summary>
-      /// <returns></returns>
-      public IDictionaryEnumerator GetEnumerator() { return new BlueRowDictionaryEnumerator(this); }
-
-      IEnumerator IEnumerable.GetEnumerator() { return new BlueRowEnumerator(this); }
-
-      public void Remove(object key) { throw new NotSupportedException("The IBlueRow is readonly"); }
-      #endregion
-   }
-   #endregion
-
-   [DataStreamUserRespresentation("ProjectStream")]
-   public class NewProjectStream : ViewStream, IStreamMapper {
-
-      private ViewStream _stream;
-      private string _project_expression;
-      private ProjectData _project_data = null;
-      //public ProjectData 
-      public NewProjectStream(ViewStream vs, string project_expression) {
-         _stream = vs;
-         _project_expression = project_expression;
-         _stream = new MapStream(vs, this);
-         mySetRealSource(_stream);
-      }
-
-      public IBlueTableSchema getOutputSchema(IBlueTableSchema input_schema) {
-         _project_data = _create_project_data(input_schema, _project_expression);
-         return _project_data.out_schema;
-      }
-
-      public IBlueRow map(IBlueRow row) {
-         return new NewProjectRow(_project_data, row);
-      }
-
-      private static ProjectData _create_project_data(IBlueTableSchema input_schema,
-                                                      string project_expression) {
-         project_expression = project_expression.Trim();
-         string[] project_toks = project_expression.Split(' ');
-         HashSet<string> removed_cols = new HashSet<string>();
-         Dictionary<string, string> mapped_to_orig_map = new Dictionary<string, string>();
-         Dictionary<string, string> orig_to_mapped = new Dictionary<string, string>();
-         // by default every col will show up, so if we have the projection as (a:a1)
-         // it will convert a to a1, 
-         // examples: if a schema has cols a b c
-         // project(parent_str, "a:aa ") -> aa b c
-         // project(parent_str, "a:aa !c") -> aa b
-         // project(parent_str, "a:aa !") -> aa
-         // project(parent_str, "a:aa b")) -> aa b
-         bool hide_all = false;
-         foreach (string p_tok in project_toks) {
-            if (p_tok.StartsWith("!")) {
-               if (p_tok.Length == 1) {
-                  // if it is a "blanket "!"=> hide all cols except the one that
-                  // are stated explicitly
-                  hide_all = true;
-               } else {
-                  string removed_col = p_tok.Substring(1);
-                  removed_cols.Add(removed_col);
-               }
-            } else if (p_tok.Contains(":")) {
-               string[] map_toks = p_tok.Split(":");
-               string orig_col = map_toks[0];
-               string mapped_col = map_toks[1];
-               mapped_to_orig_map.Add(mapped_col, orig_col);
-               orig_to_mapped.Add(orig_col, mapped_col);
-            } else {
-               if (input_schema.columnNames.Contains(p_tok)) {
-                  string orig_col = p_tok;
-                  string mapped_col = orig_col;
-                  orig_to_mapped.Add(orig_col, mapped_col);
-               }
-            }
-         }
-
-         // make the new schema
-         List<ColumnInfo> new_cols = new List<ColumnInfo>();
-         foreach (ColumnInfo ci in input_schema.columnInfo) {
-            if (ci.colName == "id") {
-               ColumnInfo nc = new ColumnInfo(ci.colName, new_cols.Count, ci.colType);
-               new_cols.Add(nc);
-               continue;
-            }
-            if (removed_cols.Contains(ci.colName)) {
-               continue;
-            } else if (orig_to_mapped.ContainsKey(ci.colName)) {
-               string mapped_name = orig_to_mapped[ci.colName];
-               ColumnInfo nc = new ColumnInfo(mapped_name, new_cols.Count, ci.colType);
-               new_cols.Add(nc);
-            } else {
-               if (hide_all)
-                  continue;
-               //if (ci.colName == "id" || removed_cols.Count > 0) {
-               // unless columns are  explicitely hidden, all columns show up
-               ColumnInfo nc = new ColumnInfo(ci.colName, new_cols.Count, ci.colType);
-               new_cols.Add(nc);
-               //}
-            }
-         }
-         IBlueTableSchema new_schema = new BlueTableSchema(input_schema.tableName, new_cols.ToArray());
-         return new ProjectData(input_schema, new_schema, mapped_to_orig_map);
-      }
-
-   }
-}

src/DataStreamSuite/DataStream/BlueOrb/SchemaGuesserStream.cs

@@ -1,30 +0,0 @@
-using BCG;
-
-namespace bts.BlueOrb {
-   public class SchemaGuesserStream : ViewStream {
-
-      private IBlueRowCollection _rows = null;
-      private ViewStream _vs;
-      public SchemaGuesserStream(ViewStream vs) {
-         _vs = vs;
-      }
-
-      protected override void myOnFirstSubscriber() {
-         _vs.onChange += _on_data;
-      }
-      protected override void myOnLastSubscriber() {
-         _vs.onChange -= _on_data;
-      }
-
-      private void _on_data(object obj) {
-         DataPacket dp = obj as DataPacket;
-         if (_rows == null) {
-            _rows = BlueDataUtility.get_typed_table(dp.newRows);
-         }
-         var dp_out = BlueDataUtility.build_dp_inc(_rows, myLastValue as DataPacket);
-         myFireEvent(dp_out);
-
-      }
-
-   }
-}

src/DataStreamSuite/DataStream/BlueOrb/SingleRowMathStream.cs

@@ -1,67 +0,0 @@
-using Bts.ArithmeticExpressions;
-
-using System;
-using System.Collections.Generic;
-using System.Linq;
-using System.Text;
-using BCG;
-
-namespace bts.BlueOrb {
-
-   [DataStreamUserRespresentation("SingleRowMathStream")]
-   public class MathStream : ViewStream, IStreamMapper {
-      private IArithmeticProgram myExp;
-      private BooleanExpressionFilter _bool;
-      List<string> myExtraColumns = new List<string>();
-      BlueTableSchema mySchema;
-
-
-      public MathStream(ViewStream vs, string mathExpression)
-         : this(vs, mathExpression, string.Empty) {
-      }
-
-      public MathStream(ViewStream vs, string mathExpression, string boolean_expression) {
-         myExp = ArithmeticProgramParser.Parse(mathExpression) ??
-            throw new Exception($"Failed to parse ArithmeticExpression '{mathExpression}'");
-         myExtraColumns = myExp.OutputIdentifiers.ToList();
-         if (!string.IsNullOrEmpty(boolean_expression))
-            _bool = new BooleanExpressionFilter(boolean_expression);
-         mySetRealSource(new MapStream(vs, this));
-      }
-
-      #region IStreamMapper Members
-
-      public IBlueTableSchema getOutputSchema(IBlueTableSchema inputSchema) {
-         if (mySchema == null) {
-            Dictionary<string, ColumnInfo> colIdx = new Dictionary<string, ColumnInfo>();
-            foreach (ColumnInfo ci in inputSchema.columnInfo)
-               colIdx[ci.colName] = ci;
-
-            foreach (string ncn in myExtraColumns) {
-               ColumnInfo newC;
-               if (colIdx.ContainsKey(ncn))
-                  newC = new ColumnInfo(colIdx[ncn].colName, colIdx[ncn].colNo, typeof(double));
-               else
-                  newC = new ColumnInfo(ncn, colIdx.Count, typeof(double));
-               colIdx[ncn] = newC;
-            }
-            mySchema = new BlueTableSchema(colIdx.Values.ToArray(), "math(" + inputSchema.tableName + ")");
-         }
-         return mySchema;
-      }
-
-      #endregion
-
-      #region IDynamicStreamMapper Members
-
-      public IBlueRow map(IBlueRow row) {
-         MutableBlueRow result = new MutableBlueRow(mySchema, row);
-         if (_bool == null || _bool.filter(row)) {
-            myExp.Evaluate(result.AsDictionary());
-         }
-         return result;
-      }
-
-      #endregion
-   }
-}

src/DataStreamSuite/DataStream/BlueOrb/TopStream.cs

@@ -1,11 +0,0 @@
-namespace bts.BlueOrb {
-   using BCG;
-   [DataStreamUserRespresentation("TopStream")]
-   public class TopStream : ViewStream { 
-      public TopStream(ViewStream vs, int n_rows, string sort_col) {
-         ViewStream rs = new BCG.TopStream (vs, n_rows, sort_col);
-         mySetRealSource(rs);
-      }
-   }
-
-}

src/DataStreamSuite/DataStream/BlueOrb/UnionStream.cs

@@ -1,261 +0,0 @@
-using System;
-using System.Collections.Generic;
-
-using mbts;
-using BCG;
-
-namespace bts.BlueOrb {
-   internal class MutableAppendTable : IBlueRowCollection {
-      private class MyRow : IBlueRow {
-         private IBlueRow myRow;
-         private int myID;
-
-         public MyRow(IBlueRow r, int id) {
-            myRow = r;
-            myID = id;
-         }
-
-         #region IBlueRow Members
-
-         public IBlueTableSchema schema { get { return myRow.schema; } }
-
-         public object this[int index] {
-            get {
-               if (myRow.schema[index].colName == "id")
-                  return myID;
-               else
-                  return myRow[index];
-            }
-         }
-
-         public object this[string colName] { get { return colName == "id" ? (object)myID : myRow[colName]; } }
-
-         #endregion
-
-         #region IDictionary Members
-
-         public void Add(object key, object value) { throw new Exception("Row is read only."); }
-
-         public void Clear() { throw new Exception("Row is read only."); }
-
-         public bool Contains(object key) { return myRow.Contains(key); }
-
-         public System.Collections.IDictionaryEnumerator GetEnumerator() { return new BlueRowDictionaryEnumerator(this); }
-
-         public bool IsFixedSize { get { return true; } }
-
-         public bool IsReadOnly { get { return true; } }
-
-         public System.Collections.ICollection Keys { get { return myRow.Keys; } }
-
-         public void Remove(object key) { throw new Exception("The method or operation is not implemented."); }
-
-         public System.Collections.ICollection Values {
-            get {
-               object[] vals = new object[myRow.Count];
-               CopyTo(vals, 0);
-               return vals;
-            }
-         }
-
-         public object this[object key] {
-            get { return key is string ? this[key as string] : this[(int)key]; }
-            set { throw new Exception("The method or operation is not implemented."); }
-         }
-
-         #endregion
-
-         #region ICollection Members
-
-         public void CopyTo(Array array, int index) {
-            myRow.CopyTo(array, index);
-            array.SetValue(myID, index + myRow.schema["id"].colNo);
-         }
-
-         public int Count { get { return myRow.Count; } }
-
-         public bool IsSynchronized { get { return myRow.IsSynchronized; } }
-
-         public object SyncRoot { get { return myRow.SyncRoot; } }
-
-         #endregion
-
-         #region IEnumerable Members
-
-         System.Collections.IEnumerator System.Collections.IEnumerable.GetEnumerator() { return new BlueRowEnumerator(this); }
-
-         #endregion
-      }
-
-      private BlueTable myRows;
-      private int myID = 0;
-      /// <summary>
-      /// maps the table no to a dictionary containing the
-      /// local IDs to a row in the final big table
-      /// </summary>
-      private Dictionary<int, Dictionary<int, IBlueRow>> myIDMap = new Dictionary<int, Dictionary<int, IBlueRow>>();
-
-      public MutableAppendTable(IBlueTableSchema schema) { myRows = new BlueTable(schema); }
-
-      public IBlueRow addRow(IBlueRow r, int tableNo) {
-         if (!myIDMap.ContainsKey(tableNo))
-            myIDMap.Add(tableNo, new Dictionary<int, IBlueRow>());
-         IBlueRow nr = new MyRow(r, myID++); // BlueDataUtility.getNewReplacedRow(r, myID++, "id");
-         myRows.addRow(nr);
-         myIDMap[tableNo].Add((int)r["id"], nr);
-         return nr;
-      }
-
-      public IBlueRow removeRow(IBlueRow r, int tableNo) {
-         IBlueRow dr = myIDMap[tableNo][(int)r["id"]];
-         myRows.deleteRow(dr);
-         myIDMap[tableNo].Remove((int)r["id"]);
-         return dr;
-      }
-
-      public IBlueRowCollection removeTable(int tableNo) {
-         if (!myIDMap.ContainsKey(tableNo))
-            return null;
-         else {
-            BlueTable rv = new BlueTable(myRows.schema);
-            foreach (IBlueRow r in myIDMap[tableNo].Values)
-               rv.addRow(r);
-            myRows.deleteRows(rv);
-            myIDMap.Remove(tableNo);
-            return rv;
-         }
-      }
-
-      public void clear() {
-         myRows.clear();
-         myIDMap.Clear();
-      }
-
-      #region IBlueRowCollection Members
-
-      public IBlueRow this[int id] { get { return myRows[id]; } }
-
-      public IBlueTableSchema schema { get { return myRows.schema; } }
-
-      public bool containsID(int id) { return myRows.containsID(id); }
-
-      #endregion
-
-      #region ICollection Members
-
-      public void CopyTo(Array array, int index) { myRows.CopyTo(array, index); }
-
-      public int Count { get { return myRows.Count; } }
-
-      public bool IsSynchronized { get { return myRows.IsSynchronized; } }
-
-      public object SyncRoot { get { return myRows.SyncRoot; } }
-
-      #endregion
-
-      #region IEnumerable Members
-
-      public System.Collections.IEnumerator GetEnumerator() { return myRows.GetEnumerator(); }
-
-      #endregion
-   }
-   /// <summary>
-   /// MutableStream can append multiple streams
-   /// as well as give the user the flexibility
-   /// to append (or remove) more streams at
-   /// runtime
-   /// </summary>
-   }
-namespace bts.BlueOrb { 
-   [DataStreamUserRespresentation("UnionStream")]
-   public class MutableAppendStream : ViewStream {
-      private bool mySubscriptionActive = false;
-      private MutableAppendTable myTable;
-      private int myMovingIndex;
-      private Dictionary<int, Pair<ViewStream, ViewStream>> myStreamIndex = new Dictionary<int, Pair<ViewStream, ViewStream>>();
-      private bool _allign_col_orders = true;
-
-      public MutableAppendStream(params ViewStream[] streams)
-         : this(new List<ViewStream>(streams)) { }
-
-      public MutableAppendStream(bool allign_col_order, params ViewStream[] streams)
-         : this(allign_col_order, new List<ViewStream>(streams)) { }
-
-      public MutableAppendStream(ICollection<ViewStream> streams)
-         : this(true, streams) { }
-
-      public MutableAppendStream(bool allign_col_orders, ICollection<ViewStream> streams) {
-         _allign_col_orders = allign_col_orders;
-         foreach (ViewStream vs in streams) {
-            myMovingIndex++;
-            ViewStream pj = new ChangeTableNameStream(vs, myMovingIndex.ToString());
-            if (_allign_col_orders)
-               pj = new ProjectStream(pj, "!");
-            myStreamIndex.Add(myMovingIndex, new Pair<ViewStream, ViewStream>(vs, pj));
-         }
-      }
-
-      public void addStream(ViewStream vs) {
-         myMovingIndex++;
-         ///the change name is done to identify the callbacks, and the project to
-         ///make all the schemas col orders the same
-         ViewStream pj = new ChangeTableNameStream(vs, myMovingIndex.ToString());
-         if (_allign_col_orders)
-            pj = new ProjectStream(pj, "!");
-         myStreamIndex.Add(myMovingIndex, new Pair<ViewStream, ViewStream>(vs, pj));
-         if (mySubscriptionActive)
-            pj.onChange += new ChangeDelegate(myHandleStreamChanges);
-      }
-
-      public void removeStream(ViewStream vs) {
-         int index = -1;
-         foreach (KeyValuePair<int, Pair<ViewStream, ViewStream>> p in myStreamIndex) {
-            if (p.Value.first == vs) {
-               index = p.Key;
-               if (mySubscriptionActive) {
-                  p.Value.second.onChange -= new ChangeDelegate(myHandleStreamChanges);
-                  if (myTable != null) {
-                     IBlueRowCollection delRows = myTable.removeTable(p.Key);
-                     if (delRows != null)
-                        myFireEvent(new DataPacket(myTable, new BlueTable(myTable.schema), delRows));
-                  }
-               }
-               break;
-            }
-            if (index > 0)
-               myStreamIndex.Remove(index);
-         }
-      }
-
-      protected override void myOnFirstSubscriber() {
-         mySubscriptionActive = true;
-         foreach (KeyValuePair<int, Pair<ViewStream, ViewStream>> p in myStreamIndex)
-            p.Value.second.onChange += new ChangeDelegate(myHandleStreamChanges);
-      }
-
-      protected override void myOnLastSubscriber() {
-         mySubscriptionActive = false;
-         foreach (KeyValuePair<int, Pair<ViewStream, ViewStream>> p in myStreamIndex)
-            p.Value.second.onChange -= new ChangeDelegate(myHandleStreamChanges);
-         if (myTable != null)
-            myTable.clear();
-      }
-
-      private void myHandleStreamChanges(object newValue) {
-         DataPacket dp = newValue as DataPacket;
-         myTable = myTable ?? new MutableAppendTable(new BlueTableSchema("MutableAppend", dp.schema.columnInfo));
-         BlueTable adds = new BlueTable(myTable.schema),
-                     dels = new BlueTable(myTable.schema);
-         int index = Int32.Parse(dp.schema.tableName);
-         foreach (IBlueRow r in dp.deletedRows) {
-            IBlueRow dr = myTable.removeRow(r, index);
-            dels.addRow(dr);
-         }
-         foreach (IBlueRow r in dp.addedRows) {
-            IBlueRow nr = myTable.addRow(r, index);
-            adds.addRow(nr);
-         }
-         myFireEvent(new DataPacket(myTable, adds, dels));
-      }
-   }
-}

src/DataStreamSuite/DataStream/DataStream.csproj

@@ -1,7 +1,7 @@
 <Project Sdk="Microsoft.NET.Sdk">
 
   <PropertyGroup>
-    <TargetFramework>net8.0</TargetFramework>
+    <TargetFramework>net7.0</TargetFramework>
     <Nullable>enable</Nullable>
   </PropertyGroup>
 
@@ -18,13 +18,11 @@
 
   <ItemGroup>
     <PackageReference Include="HtmlAgilityPack" Version="1.11.57" />
-    <PackageReference Include="IronPython" Version="3.4.2" />
     <PackageReference Include="Microsoft.Data.Analysis" Version="0.21.1" />
     <PackageReference Include="MongoDB.Bson" Version="2.23.1" />
     <PackageReference Include="MongoDB.Driver" Version="2.23.1" />
     <PackageReference Include="Parquet.Net" Version="5.0.1" />
     <PackageReference Include="System.Data.SQLite" Version="1.0.118" />
-    <PackageReference Include="MathNet.Numerics" Version="5.0.0" />
   </ItemGroup>
 
 </Project>

src/DataStreamSuite/DataStream/Primitives/Count/CountStream.cs

@@ -1,47 +1,52 @@
 using System;
 
-namespace bts.BlueOrb {
-   using BCG;
-   [DataStreamUserRespresentation("CountStream")]
-   public class CountStream : ViewStream {
+namespace BCG
+{
+   [DataStreamUserRespresentation ("count")]
+	public class CountStream : ViewStream
+	{
       private DataPacket myLastFiredDataPacket;
-      private IBlueTableSchema mySchema = new BlueTableSchema("count",
-         new ColumnInfo("id", 0, typeof(int)),
-         new ColumnInfo("count", 1, typeof(int)));
+      private IBlueTableSchema mySchema = new BlueTableSchema ("count", 
+         new ColumnInfo ("id", 0, typeof(int)),
+         new ColumnInfo ("count", 1, typeof(int)));
       private int myId = 0;
       private ViewStream myVS;
 
-      public CountStream(ViewStream vs) {
+      public CountStream (ViewStream vs)
+      {
          myVS = vs;
       }
 
-      public CountStream(string tableName)
-         : this(new TableStream(tableName)) { }
+      public CountStream (string tableName)
+         : this (new TableStream (tableName))
+      {}
 
-      protected override void myOnFirstSubscriber() {
+      protected override void myOnFirstSubscriber () {
          myVS.onChange += new ChangeDelegate(myHandleChanges);
       }
-      protected override void myOnLastSubscriber() {
+      protected override void myOnLastSubscriber () {
          myVS.onChange -= new ChangeDelegate(myHandleChanges);
       }
 
 
-      private void myHandleChanges(object newValue) {
-         myFireEvent(
+      private void myHandleChanges(object newValue)
+      {
+         myFireEvent (
             myLastFiredDataPacket =
                myLastFiredDataPacket == null ?
-                  new DataPacket(myGetRows(newValue, ++myId), myGetRows(newValue, myId),
-                  new BlueTable(mySchema))
+                  new DataPacket (myGetRows (newValue, ++myId), myGetRows (newValue, myId), 
+                  new BlueTable (mySchema))
                   :
-                  new DataPacket(myGetRows(newValue, ++myId), myGetRows(newValue, myId),
+                  new DataPacket (myGetRows (newValue, ++myId), myGetRows (newValue, myId), 
                   myLastFiredDataPacket.newRows)
             );
       }
 
-      private IBlueRowCollection myGetRows(object dp, int id) {
-         BlueTable retVal = new BlueTable(mySchema);
+      private IBlueRowCollection myGetRows (object dp, int id)
+      {
+         BlueTable retVal = new BlueTable (mySchema);
          retVal.addRow
-            (new BlueRow(mySchema, id, (dp as DataPacket).newRows.Count));
+            (new BlueRow (mySchema, id, (dp as DataPacket).newRows.Count));
          return retVal;
       }
 

src/DataStreamSuite/DataStream/Primitives/Filter/FilterStream.cs

@@ -5,20 +5,25 @@ using mbts.config;
 using System;
 using System.Text.RegularExpressions;
 
-namespace BCG.StreamWorld.Addition {
-   [DataStreamUserRespresentation(false)]
-   public class FilterStream : ViewStream {
+namespace BCG.StreamWorld.Addition
+{
+   [DataStreamUserRespresentation (false)]
+   public class FilterStream : ViewStream
+   {
       private ViewStream myBaseStream;
       private BlueTable myBaseTable, myStateTable;
       private IStreamFilter myFilter;
 
       public FilterStream(ViewStream vs)
-         : this(vs, "1=1") { }
+         : this(vs, "1=1")
+      { }
 
       public FilterStream(ViewStream vs, string filterExp)
-         : this(vs, new BooleanExpressionFilter(filterExp)) { }
+         : this(vs, new BooleanExpressionFilter(filterExp))
+      { }
 
-      public FilterStream(ViewStream vs, IStreamFilter filter) {
+      public FilterStream(ViewStream vs, IStreamFilter filter)
+      {
          myBaseStream = vs;
          myFilter = filter;
          if (filter is IDynamicStreamFilter)
@@ -26,35 +31,43 @@ namespace BCG.StreamWorld.Addition {
       }
 
 
-      protected override void myOnFirstSubscriber() {
+      protected override void myOnFirstSubscriber()
+      {
          myBaseStream.onChange += new ChangeDelegate(myHandleChanges);
       }
 
-      protected override void myOnLastSubscriber() {
+      protected override void myOnLastSubscriber()
+      {
          myBaseStream.onChange -= new ChangeDelegate(myHandleChanges);
          myBaseTable.clear();
          myStateTable.clear();
       }
 
-      private void myHandleChanges(object newValue) {
+      private void myHandleChanges(object newValue)
+      {
          DataPacket dp = newValue as DataPacket;
-         if (myStateTable == null) {
+         if (myStateTable == null)
+         {
             bts.Log.assert(myBaseTable == null);
             myStateTable = new BlueTable(dp.schema);
             myBaseTable = new BlueTable(dp.schema);
          }
          BlueTable dels = new BlueTable(dp.schema),
                      adds = new BlueTable(dp.schema);
-         foreach (IBlueRow r in dp.deletedRows) {
+         foreach (IBlueRow r in dp.deletedRows)
+         {
             myBaseTable.deleteRow(r);
-            if (myStateTable.containsID((int)r["id"])) {
+            if (myStateTable.containsID((int)r["id"]))
+            {
                dels.addRow(r);
                myStateTable.deleteRow(r);
             }
          }
-         foreach (IBlueRow r in dp.addedRows) {
+         foreach (IBlueRow r in dp.addedRows)
+         {
             myBaseTable.addRow(r);
-            if (myFilter.filter(r)) {
+            if (myFilter.filter(r))
+            {
                adds.addRow(r);
                myStateTable.addRow(r);
             }
@@ -63,10 +76,13 @@ namespace BCG.StreamWorld.Addition {
             myFireEvent(new DataPacket(myStateTable, adds, dels));
       }
 
-      public IStreamFilter filter {
-         set {
+      public IStreamFilter filter
+      {
+         set
+         {
             myFilter = value;
-            if (myBaseTable != null) {
+            if (myBaseTable != null)
+            {
                IBlueRowCollection oldState = new BlueTable(myStateTable);
                myStateTable.clear();
                foreach (IBlueRow r in myBaseTable)

src/DataStreamSuite/DataStream/Primitives/GroupBy/GroupByBundle.cs

@@ -4,46 +4,42 @@ namespace bts.SilverPine {
    using System;
    using System.Collections.Generic;
    using System.Linq;
-   //using MathNet.Numerics.Statistics;
 
+   using ZstdSharp.Unsafe;
 
-   public enum MultiRowOp {
+   public enum GroupByBundleOp {
       Mean,
       Median,
       Mode,
-      Range,
       StdDev,
       Max,
       Min,
       First,
       Last,
-      Sum,
-      EdgeDiff,
-      EdgeDiffPercentage,
-      //IsEdgeDiff,
+      Sum
    }
-   public class MultiRowOpHelper {
-      public readonly List<Tuple<string, MultiRowOp>> operations;
+   public class GroupByBundle {
+      public readonly List<Tuple<string, GroupByBundleOp>> operations;
 
-      public static MultiRowOpHelper parse(string text) {
+      public static GroupByBundle parse(string text) {
          // text is inp_col: op, inp_col: op, inp_col:op
-         List<Tuple<string, MultiRowOp>> rv = new List<Tuple<string, MultiRowOp>>();
+         List<Tuple<string, GroupByBundleOp>> rv = new List<Tuple<string, GroupByBundleOp>>();
          string[] calcs = text.Split(',');
          foreach (string c in calcs) {
             string[] toks = c.Split(':');
             string inp_col = toks[0].Trim();
             string op_txt = toks[1].Trim();
-            MultiRowOp op = Enum.Parse<MultiRowOp>(op_txt, true);
+            GroupByBundleOp op = Enum.Parse<GroupByBundleOp>(op_txt, true);
             rv.Add(Tuple.Create(inp_col, op));
          }
-         return new MultiRowOpHelper(rv);
+         return new GroupByBundle(rv);
       }
 
-      public MultiRowOpHelper(List<Tuple<string, MultiRowOp>> operations_) {
+      public GroupByBundle(List<Tuple<string, GroupByBundleOp>> operations_) {
          operations = operations_;
       }
 
-      public static IBlueRow group_by(List<IBlueRow> rows, MultiRowOpHelper parameters) {
+      public static IBlueRow group_by(List<IBlueRow> rows, GroupByBundle parameters) {
          if (rows.Count() == 0) {
             return null;
          }
@@ -51,8 +47,8 @@ namespace bts.SilverPine {
          IBlueRow ir = rows.ElementAt(0);
          IBlueTableSchema in_schema = ir.schema;
          IBlueTableSchema out_schema = in_schema;
-         IDictionary<string, Tuple<string, MultiRowOp>> out_col_map =
-            new Dictionary<string, Tuple<string, MultiRowOp>>();
+         IDictionary<string, Tuple<string, GroupByBundleOp>> out_col_map =
+            new Dictionary<string, Tuple<string, GroupByBundleOp>>();
          foreach (var t in parameters.operations) {
             string out_col_name = $"{t.Item1}_{t.Item2}".ToLower();
             out_schema = out_schema.add_column(TypeCode.Double, out_col_name);
@@ -62,7 +58,7 @@ namespace bts.SilverPine {
          foreach (var kv in out_col_map) {
             string out_col_name = kv.Key;
             string in_col_name = kv.Value.Item1;
-            MultiRowOp op = kv.Value.Item2;
+            GroupByBundleOp op = kv.Value.Item2;
             IEnumerable<double> col_vals = BlueDataUtility.get_colvalues<double>(rows, in_col_name);
             double comp_res = compute_op(col_vals, op);
             or.setValue(out_col_name, comp_res);
@@ -70,51 +66,20 @@ namespace bts.SilverPine {
          return or;
       }
 
-      // rolling_rows are the rolling window of rows. the 0th row is the one thats gettin
-      // calculated. rows 1 - n-1 are its window rows. the schema is based on the operations.
-
-      public IBlueRow rolling_calc(List<IBlueRow> rolling_rows) {
-         if (rolling_rows.Count() == 0) {
-            return null;
-         }
-         IBlueRow ir = rolling_rows.ElementAt(0);
-         IBlueTableSchema in_schema = ir.schema;
-         IBlueTableSchema out_schema = in_schema;
-         IDictionary<string, Tuple<string, MultiRowOp>> out_col_map =
-            new Dictionary<string, Tuple<string, MultiRowOp>>();
-         foreach (var t in operations) {
-            string out_col_name = $"{t.Item1}_{t.Item2}".ToLower();
-            out_schema = out_schema.add_column(TypeCode.Double, out_col_name);
-            out_col_map[out_col_name] = t;
-         }
-         MutableBlueRow or = new MutableBlueRow(out_schema, ir);
-         foreach (var kv in out_col_map) {
-            string out_col_name = kv.Key;
-            string in_col_name = kv.Value.Item1;
-            MultiRowOp op = kv.Value.Item2;
-            IEnumerable<double> col_vals = BlueDataUtility.get_colvalues<double>(rolling_rows, in_col_name);
-            double comp_res = compute_op(col_vals, op);
-            or.setValue(out_col_name, comp_res);
-         }
-         return or;
-      }
-
-      public static double compute_op(IEnumerable<double> col_vals, MultiRowOp op) {
+      public static double compute_op(IEnumerable<double> col_vals, GroupByBundleOp op) {
          switch (op) {
-            case MultiRowOp.Max:
+            case GroupByBundleOp.Max:
                return col_vals.Max<double>();
-            case MultiRowOp.Min:
+            case GroupByBundleOp.Min:
                return col_vals.Min<double>();
-            case MultiRowOp.Mean:
+            case GroupByBundleOp.Mean:
                return col_vals.Sum() / col_vals.Count();
-            case MultiRowOp.Sum:
+            case GroupByBundleOp.Sum:
                return col_vals.Sum();
-            case MultiRowOp.First:
+            case GroupByBundleOp.First:
                return col_vals.First();
-            case MultiRowOp.Last:
+            case GroupByBundleOp.Last:
                return col_vals.Last();
-            case MultiRowOp.StdDev:
-               return MathNet.Numerics.Statistics.Statistics.StandardDeviation(col_vals);
             default:
                // TODO
                Log.assert(false);

src/DataStreamSuite/DataStream/Primitives/GroupBy/GroupByStream.cs

@@ -5,7 +5,7 @@ using System.Collections.Generic;
 using System.Linq;
 
 namespace BCG {
-   internal class GroupByStream : MethodDrivenStream {
+   public class GroupByStream : MethodDrivenStream {
       private class MyFuncGroupByEvaluator : IGroupByEvaluator {
          private string[] _groupby_cols;
          private Func<DataPacket, IBlueRow> _groupby_func;

src/DataStreamSuite/DataStream/Primitives/GroupBy/NonIncremental/AggregationSteam.cs

@@ -1,20 +0,0 @@
-namespace bts.SilverPine {
-   using BCG;
-
-   using System;
-
-   [DataStreamUserRespresentation("AggregationStream")]
-   public class AgregationStream : ViewStream {
-
-      public AgregationStream(ViewStream parent_str, string operation_txt) {
-         ViewStream rs = new NonIncGroupByStream(parent_str, String.Empty, operation_txt);
-         mySetRealSource(rs);
-      }
-
-      public AgregationStream(ViewStream parent_str, string groupby_col, string operation_txt) {
-         ViewStream rs = new NonIncGroupByStream(parent_str, groupby_col, operation_txt);
-         mySetRealSource(rs);
-      }
-   }
-
-}

src/DataStreamSuite/DataStream/Primitives/GroupBy/NonIncremental/NonIncGroupByStream.cs

@@ -8,7 +8,7 @@ using bts;
 using bts.SilverPine;
 
 namespace BCG {
-   [DataStreamUserRespresentation("AggregateStream")]
+   [DataStreamUserRespresentation("aggregate")]
    public class NonIncGroupByStream : ViewStream {
       private class DelNonIncGroupByEvaluator : INonIncGroupByEvaluator {
          Func<string[]> _groupby_cols_func;
@@ -53,18 +53,15 @@ namespace BCG {
                                  Func<List<IBlueRow>, List<IBlueRow>> evaluate_func)
                                  : this(main_stream, new DelNonIncGroupByEvaluator(groupby_cols_func, evaluate_func)) { }
 
-      public NonIncGroupByStream(ViewStream parent_str, string groupby_col, string standard_group_by_text)
-      : this(parent_str, groupby_col, standard_group_by_text, String.Empty) { }
-
-      public NonIncGroupByStream(ViewStream parent_str, string groupby_col, string standard_group_by_text, string optional_sort_col) {
-         MultiRowOpHelper std_group_by = MultiRowOpHelper.parse(standard_group_by_text);
+      public NonIncGroupByStream(ViewStream parent_str, string groupby_col, string standard_group_by_text, string optional_sort_col = null) {
+         GroupByBundle std_group_by = GroupByBundle.parse(standard_group_by_text);
          Func<string[]> groupby_cols_func = () => {
             return String.IsNullOrEmpty(groupby_col) ? null : new string[] { groupby_col };
          };
          Func<List<IBlueRow>, List<IBlueRow>> evaluate_func = (group_rows) => {
             if (!String.IsNullOrEmpty(optional_sort_col))
                group_rows.Sort((r1, r2) => BlueDataUtility.compare_rows(r2, r1, optional_sort_col));
-            IBlueRow rv = MultiRowOpHelper.group_by(group_rows, std_group_by);
+            IBlueRow rv = GroupByBundle.group_by(group_rows, std_group_by);
             return new IBlueRow[] { rv }.ToList();
          };
          ViewStream rs = new NonIncGroupByStream(parent_str, groupby_cols_func, evaluate_func);

src/DataStreamSuite/DataStream/Primitives/Join/NonIncremental/NonIncJoinBookkeeper.cs

@@ -74,7 +74,7 @@ namespace BCG {
       }
 
       public void _on_stream_cb(object dp_obj) {
-         //Log.Gen.info($"NonIncJoinBookkeeper._on_stream_cb dp_obj: {dp_obj}");
+         Log.Gen.info($"NonIncJoinBookkeeper._on_stream_cb dp_obj: {dp_obj}");
          var timer = Stopwatch.StartNew();
          DataPacket dp = (DataPacket)dp_obj;
          _fill_missing_data(dp);

src/DataStreamSuite/DataStream/Primitives/Join/NonIncremental/NonIncJoinStream.cs

@@ -50,7 +50,7 @@ namespace BCG {
          int data_col_no = 1;
          for (int jp = 0; jp < _join_specs.Length; jp++) {
             NonIncJoinBookkeeper s = new NonIncJoinBookkeeper(_join_specs[jp], jp, data_col_no, _row_hasher);
-            //Log.Gen.info($"NonIncJoinStream.myOnFirstSubscriber jp: {jp} spec: {_join_specs[jp]}");
+            Log.Gen.info($"NonIncJoinStream.myOnFirstSubscriber jp: {jp} spec: {_join_specs[jp]}");
             data_col_no += _join_specs[jp].data_col_names.Length;
             s.on_new_data += _stream_callback;
             _book_keeping_data.Add(s);
@@ -71,7 +71,7 @@ namespace BCG {
       }
 
       private void _stream_callback() {
-         //Log.Gen.info("NonIncJoinStream._stream_callback received callback");
+         Log.Gen.info("NonIncJoinStream._stream_callback received callback");
          _is_dirty = true;
          doWork();
       }
@@ -99,7 +99,7 @@ namespace BCG {
 
       private int _n_total_fires = 0, _n_expensive_fires = 0;
       private void _form_and_fire() {
-         //Log.Gen.info($"NonIncJoinStream._form_and_fire _is_ready(): {_is_ready()}");
+         Log.Gen.info($"NonIncJoinStream._form_and_fire _is_ready(): {_is_ready()}");
          if (_is_ready()) {
             _construct_schema();
             var timer = Stopwatch.StartNew();
@@ -126,11 +126,11 @@ namespace BCG {
             _n_total_fires++;
             //if (timer.ElapsedMilliseconds >= 250) {
             _n_expensive_fires++;
-            //string log_msg = $"Join and map time for {_book_keeping_data.Count} "
-            //   + $"tables {timer.Elapsed.TotalMilliseconds} msecs, ratio {_n_expensive_fires}/{_n_total_fires}";
-            //Log.Gen.info(log_msg);
+            string log_msg = $"Join and map time for {_book_keeping_data.Count} "
+               + $"tables {timer.Elapsed.TotalMilliseconds} msecs, ratio {_n_expensive_fires}/{_n_total_fires}";
+            Log.Gen.info(log_msg);
             //}
-            var dp_out = BlueDataUtility.build_dp_inc(dp_output_rows, myLastValue as DataPacket, true);
+            var dp_out = BlueDataUtility.build_dp_inc(dp_output_rows, myLastValue as DataPacket);
             myFireEvent(dp_out);
             //myFireEvent(new DataPacket(dp_output_rows, dp_output_rows, myLastValue == null ? new BlueTable(_output_schema) : (myLastValue as DataPacket).newRows));
          }
@@ -152,7 +152,7 @@ namespace BCG {
       }
 
       public void doWork() {
-         //Log.Gen.info($"NonIncJoinStream.doWork _last_fire_time: {_last_fire_time} _is_dirty: {_is_dirty}");
+         Log.Gen.info($"NonIncJoinStream.doWork _last_fire_time: {_last_fire_time} _is_dirty: {_is_dirty}");
          if (DateTime.Now - _last_fire_time >= _throttle_period && _is_dirty) {
             _is_dirty = false;
             _last_fire_time = DateTime.Now;

src/DataStreamSuite/DataStream/Primitives/Map/GlobalMapStream.cs

@@ -69,6 +69,4 @@ namespace bts {
          myFireEvent(dp_out);
       }
    }
-
-
 }

src/DataStreamSuite/DataStream/Primitives/Map/MapStream.cs

@@ -13,8 +13,9 @@ namespace BCG {
    /// desired row ie newRow = f(row), where f is the mapping fucntion. Note that the id 
    /// is preserved across the rows, ie row["id"] = f(row)["id"]
    /// </summary>
-   [DataStreamUserRespresentation(false)]
-   public class MapStream : MethodDrivenStream {
+   [DataStreamUserRespresentation (false)]
+   public class MapStream : MethodDrivenStream
+   {
       protected IStreamMapper myMapper;
       protected IChangeableStreamMapper myChangeableMapper = null;
       private bool myUseNewMap = Configuration.get<DataStreamConfig>().UseNewMap;

src/DataStreamSuite/DataStream/Primitives/Map/RollingComputationStream

@@ -1,14 +0,0 @@
-namespace bts.SilverPine
-{
-   using BCG;
-
-   [DataStreamUserRespresentation("RollingComputationStream")]
-   public class RollingComputationStream : ViewStream
-   {
-      private ViewStream _parent_str;
-      public RollingComputationStream(ViewStream parent_str, string operation_txt, string roll_column, uint window_sz)
-      {
-
-      }
-   }
-}

src/DataStreamSuite/DataStream/Primitives/Map/RollingComputationStream.cs

@@ -1,10 +0,0 @@
-namespace bts.SilverPine {
-   using BCG;
-
-
-   [DataStreamUserRespresentation("RollingComputationStream")]
-   public class RollingComputationStream : ViewStream {
-      public RollingComputationStream(ViewStream parent_str, string operation_txt, string roll_column, uint window_sz) {
-      }
-   }
-}

src/DataStreamSuite/DataStream/Primitives/Project/New/NewProjectStream.cs

@@ -108,7 +108,6 @@ namespace bts {
 
    }
 
-   [DataStreamUserRespresentation("Project")]
    public class NewProjectStream : ViewStream, IStreamMapper {
 
       private ViewStream _stream;

src/DataStreamSuite/DataStream/Primitives/Time/SecondStream.cs

@@ -37,42 +37,4 @@ namespace BCG
       }
          
    }
-
-   [DataStreamUserRespresentation("throttle")]
-   public class ThrottleStream : ViewStream{
-      private readonly ViewStream _main_stream;
-      private readonly TimeSpan _ts;
-      private readonly ViewStreamDatabase _vsd;
-      private DateTime _last_fire_time = DateTime.MinValue;
-
-      public ThrottleStream (ViewStream main_stream, int throttle_seconds)
-      :
-      this(main_stream, TimeSpan.FromSeconds(throttle_seconds), ViewStreamDatabase.instance)
-      {
-      }
-
-      public ThrottleStream (ViewStream main_stream, TimeSpan ts, ViewStreamDatabase vsd){
-         _main_stream = main_stream;
-         _ts = ts;
-         _vsd = vsd;
-      }
-
-       protected override void myOnFirstSubscriber() {
-         _main_stream.onChange += _handle_base_stream_update;
-      }
-
-      protected override void myOnLastSubscriber() {
-         _main_stream.onChange -= _handle_base_stream_update;
-      }
-
-      private void _handle_base_stream_update(object dp_obj){
-         if (DateTime.Now - _last_fire_time >= _ts){
-            DataPacket dp_in = dp_obj as DataPacket;
-            _last_fire_time = DateTime.Now;
-            DataPacket dp = BlueDataUtility.build_dp_inc(dp_in.newRows, myLastValue as DataPacket);
-            myFireEvent(dp);
-         }
-      }
- }
-
 }

src/DataStreamSuite/DataStream/Primitives/Time/ThrottleStream

@@ -1,40 +0,0 @@
-namespace bts.SilverPine{
-   using BGC;
-   using System;
-   [DataStreamUserRespresentation("throttle")]
-   public class ThrottleStream : ViewStream{
-      private readonly ViewStream _main_stream;
-      private readonly TimeSpan _ts;
-      private readonly ViewStreamDatabase _vsd;
-      private DateTime _last_fire_time = DateTime.MinValue;
-
-      public ThrottleStream (ViewStream main_stream, int throttle_seconds)
-      :
-      this(main_stream, TimeSpan.FromSeconds(throttle_seconds), ViewStreamDatabase.instance)
-      {
-      }
-
-      public ThrottleStream (ViewStream main_stream, TimeSpan ts, ViewStreamDatabase vsd){
-         _main_stream = main_stream;
-         _ts = ts;
-         _vsd;
-      }
-
-       protected override void myOnFirstSubscriber() {
-         _main_stream.onChange += _handle_base_stream_update;
-      }
-
-      protected override void myOnLastSubscriber() {
-         _main_stream.onChange -= _handle_base_stream_update;
-      }
-
-      private void _handle_base_stream_update(object dp_obj){
-         if (DateTime.Now - _last_fire_time >= _ts){
-            DataPacket dp_in = dp_obj;
-            _last_fire_time = DateTime.Now;
-            DataPacket dp = BlueDataUtility.build_dp_inc(dp_in.newRow, myLastValue as DataPacket);
-            myFireEvent(dp);
-         }
-      }
- }
-}

src/DataStreamSuite/DataStream/Primitives/Top/TopStream.cs

@@ -40,4 +40,4 @@ namespace BCG {
             myFireSwitch = myLastFiredPacket.addedRows.Count + myLastFiredPacket.deletedRows.Count == 0 ? false : true;
         }
     }
-}
+}

src/DataStreamSuite/DataStream/Primitives/Union/MutableAppendStream.cs

@@ -1,15 +1,18 @@
 using System;
 using System.Collections.Generic;
-
 using mbts;
 
-namespace BCG {
-   internal class MutableAppendTable : IBlueRowCollection {
-      private class MyRow : IBlueRow {
+namespace BCG
+{
+   internal class MutableAppendTable : IBlueRowCollection
+   {
+      private class MyRow : IBlueRow
+      {
          private IBlueRow myRow;
          private int myID;
 
-         public MyRow(IBlueRow r, int id) {
+         public MyRow(IBlueRow r, int id)
+         {
             myRow = r;
             myID = id;
          }
@@ -18,8 +21,10 @@ namespace BCG {
 
          public IBlueTableSchema schema { get { return myRow.schema; } }
 
-         public object this[int index] {
-            get {
+         public object this[int index]
+         {
+            get
+            {
                if (myRow.schema[index].colName == "id")
                   return myID;
                else
@@ -47,17 +52,20 @@ namespace BCG {
 
          public System.Collections.ICollection Keys { get { return myRow.Keys; } }
 
-         public void Remove(object key) { throw new Exception("The method or operation is not implemented."); }
+         public void Remove(object key) {throw new Exception("The method or operation is not implemented.");}
 
-         public System.Collections.ICollection Values {
-            get {
+         public System.Collections.ICollection Values
+         {
+            get
+            {
                object[] vals = new object[myRow.Count];
                CopyTo(vals, 0);
                return vals;
             }
          }
 
-         public object this[object key] {
+         public object this[object key]
+         {
             get { return key is string ? this[key as string] : this[(int)key]; }
             set { throw new Exception("The method or operation is not implemented."); }
          }
@@ -66,7 +74,8 @@ namespace BCG {
 
          #region ICollection Members
 
-         public void CopyTo(Array array, int index) {
+         public void CopyTo(Array array, int index)
+         {
             myRow.CopyTo(array, index);
             array.SetValue(myID, index + myRow.schema["id"].colNo);
          }
@@ -81,7 +90,8 @@ namespace BCG {
 
          #region IEnumerable Members
 
-         System.Collections.IEnumerator System.Collections.IEnumerable.GetEnumerator() { return new BlueRowEnumerator(this); }
+         System.Collections.IEnumerator System.Collections.IEnumerable.GetEnumerator()
+         { return new BlueRowEnumerator(this); }
 
          #endregion
       }
@@ -96,7 +106,8 @@ namespace BCG {
 
       public MutableAppendTable(IBlueTableSchema schema) { myRows = new BlueTable(schema); }
 
-      public IBlueRow addRow(IBlueRow r, int tableNo) {
+      public IBlueRow addRow(IBlueRow r, int tableNo)
+      {
          if (!myIDMap.ContainsKey(tableNo))
             myIDMap.Add(tableNo, new Dictionary<int, IBlueRow>());
          IBlueRow nr = new MyRow(r, myID++); // BlueDataUtility.getNewReplacedRow(r, myID++, "id");
@@ -105,17 +116,20 @@ namespace BCG {
          return nr;
       }
 
-      public IBlueRow removeRow(IBlueRow r, int tableNo) {
+      public IBlueRow removeRow(IBlueRow r, int tableNo)
+      {
          IBlueRow dr = myIDMap[tableNo][(int)r["id"]];
          myRows.deleteRow(dr);
          myIDMap[tableNo].Remove((int)r["id"]);
          return dr;
       }
 
-      public IBlueRowCollection removeTable(int tableNo) {
+      public IBlueRowCollection removeTable(int tableNo)
+      {
          if (!myIDMap.ContainsKey(tableNo))
             return null;
-         else {
+         else
+         {
             BlueTable rv = new BlueTable(myRows.schema);
             foreach (IBlueRow r in myIDMap[tableNo].Values)
                rv.addRow(r);
@@ -125,7 +139,8 @@ namespace BCG {
          }
       }
 
-      public void clear() {
+      public void clear()
+      {
          myRows.clear();
          myIDMap.Clear();
       }
@@ -136,7 +151,7 @@ namespace BCG {
 
       public IBlueTableSchema schema { get { return myRows.schema; } }
 
-      public bool containsID(int id) { return myRows.containsID(id); }
+      public bool containsID(int id) {return myRows.containsID(id);}
 
       #endregion
 
@@ -154,7 +169,7 @@ namespace BCG {
 
       #region IEnumerable Members
 
-      public System.Collections.IEnumerator GetEnumerator() { return myRows.GetEnumerator(); }
+      public System.Collections.IEnumerator GetEnumerator() {return myRows.GetEnumerator();}
 
       #endregion
    }
@@ -164,8 +179,8 @@ namespace BCG {
    /// to append (or remove) more streams at
    /// runtime
    /// </summary>
-   [DataStreamUserRespresentation("UnionStream")]
-   public class MutableAppendStream : ViewStream {
+   public class MutableAppendStream : ViewStream
+   {
       private bool mySubscriptionActive = false;
       private MutableAppendTable myTable;
       private int myMovingIndex;
@@ -173,17 +188,22 @@ namespace BCG {
       private bool _allign_col_orders = true;
 
       public MutableAppendStream(params ViewStream[] streams)
-         : this(new List<ViewStream>(streams)) { }
+         : this(new List<ViewStream>(streams))
+      { }
 
       public MutableAppendStream(bool allign_col_order, params ViewStream[] streams)
-         : this(allign_col_order, new List<ViewStream>(streams)) { }
+         : this(allign_col_order, new List<ViewStream>(streams))
+      { }
 
       public MutableAppendStream(ICollection<ViewStream> streams)
-         : this(true, streams) { }
+         : this(true, streams)
+      { }
 
-      public MutableAppendStream(bool allign_col_orders, ICollection<ViewStream> streams) {
+      public MutableAppendStream(bool allign_col_orders, ICollection<ViewStream> streams)
+      {
          _allign_col_orders = allign_col_orders;
-         foreach (ViewStream vs in streams) {
+         foreach (ViewStream vs in streams)
+         {
             myMovingIndex++;
             ViewStream pj = new ChangeTableNameStream(vs, myMovingIndex.ToString());
             if (_allign_col_orders)
@@ -192,7 +212,8 @@ namespace BCG {
          }
       }
 
-      public void addStream(ViewStream vs) {
+      public void addStream(ViewStream vs)
+      {
          myMovingIndex++;
          ///the change name is done to identify the callbacks, and the project to
          ///make all the schemas col orders the same
@@ -204,14 +225,19 @@ namespace BCG {
             pj.onChange += new ChangeDelegate(myHandleStreamChanges);
       }
 
-      public void removeStream(ViewStream vs) {
+      public void removeStream(ViewStream vs)
+      {
          int index = -1;
-         foreach (KeyValuePair<int, Pair<ViewStream, ViewStream>> p in myStreamIndex) {
-            if (p.Value.first == vs) {
+         foreach (KeyValuePair<int, Pair<ViewStream, ViewStream>> p in myStreamIndex)
+         {
+            if (p.Value.first == vs)
+            {
                index = p.Key;
-               if (mySubscriptionActive) {
+               if (mySubscriptionActive)
+               {
                   p.Value.second.onChange -= new ChangeDelegate(myHandleStreamChanges);
-                  if (myTable != null) {
+                  if (myTable != null)
+                  {
                      IBlueRowCollection delRows = myTable.removeTable(p.Key);
                      if (delRows != null)
                         myFireEvent(new DataPacket(myTable, new BlueTable(myTable.schema), delRows));
@@ -224,13 +250,15 @@ namespace BCG {
          }
       }
 
-      protected override void myOnFirstSubscriber() {
+      protected override void myOnFirstSubscriber()
+      {
          mySubscriptionActive = true;
          foreach (KeyValuePair<int, Pair<ViewStream, ViewStream>> p in myStreamIndex)
             p.Value.second.onChange += new ChangeDelegate(myHandleStreamChanges);
       }
 
-      protected override void myOnLastSubscriber() {
+      protected override void myOnLastSubscriber()
+      {
          mySubscriptionActive = false;
          foreach (KeyValuePair<int, Pair<ViewStream, ViewStream>> p in myStreamIndex)
             p.Value.second.onChange -= new ChangeDelegate(myHandleStreamChanges);
@@ -238,17 +266,20 @@ namespace BCG {
             myTable.clear();
       }
 
-      private void myHandleStreamChanges(object newValue) {
+      private void myHandleStreamChanges(object newValue)
+      {         
          DataPacket dp = newValue as DataPacket;
          myTable = myTable ?? new MutableAppendTable(new BlueTableSchema("MutableAppend", dp.schema.columnInfo));
          BlueTable adds = new BlueTable(myTable.schema),
                      dels = new BlueTable(myTable.schema);
          int index = Int32.Parse(dp.schema.tableName);
-         foreach (IBlueRow r in dp.deletedRows) {
+         foreach (IBlueRow r in dp.deletedRows)
+         {
             IBlueRow dr = myTable.removeRow(r, index);
             dels.addRow(dr);
          }
-         foreach (IBlueRow r in dp.addedRows) {
+         foreach (IBlueRow r in dp.addedRows)
+         {
             IBlueRow nr = myTable.addRow(r, index);
             adds.addRow(nr);
          }

src/DataStreamSuite/DataStreamCore/BlueData/BlueDataUtility.cs

@@ -280,28 +280,6 @@ namespace BCG {
 
       #region json
 
-      public static void write_to_json_file(this IBlueRowCollection table, string file_path) { 
-         write_dict_to_json_file(file_path, table.to_namevalue_pairs());
-      }
-
-      public static void write_dict_to_json_file(string file_path, List<Dictionary<string, string>> data) {
-         var options = new JsonSerializerOptions { WriteIndented = true };
-         string json = System.Text.Json.JsonSerializer.Serialize(data, options);
-         File.WriteAllText(file_path, json);
-      }
-
-      public static List<Dictionary<string, string>> ReadFromFile(string filePath) {
-         string json = File.ReadAllText(filePath);
-         return System.Text.Json.JsonSerializer.Deserialize<List<Dictionary<string, string>>>(json);
-      }
-
-      public static void write_table_to_json(IBlueRowCollection table) {
-         List<Dictionary<string, string>> rows = new List<Dictionary<string, string>>();
-         foreach (IBlueRow row in table) {
-            row.to_json();
-         }
-      }
-
       public static Dictionary<string, string> convert_json_to_dictionary(string json_string) {
          Dictionary<string, string> dictionary = JsonConvert.DeserializeObject<Dictionary<string, string>>(json_string);
          return dictionary;
@@ -370,14 +348,6 @@ namespace BCG {
          }
          return rv;
       }
-
-      public static List<Dictionary<string,string>> to_namevalue_pairs(this IBlueRowCollection table) {
-         List<Dictionary<string, string>> rv = new List<Dictionary<string, string>>();
-         foreach (IBlueRow r in table) {
-            rv.Add(r.to_dict());
-         }
-         return rv;
-      }
       #region Schema types
       // these are the only types allowed in schemas
       private static IReadOnlyDictionary<TypeCode, Type> _typecode_to_type = new Dictionary<TypeCode, Type>
@@ -1255,20 +1225,6 @@ namespace BCG {
          return rv;
       }
 
-      public static Dictionary<string, string> to_dict(this IBlueRow r) {
-         Dictionary<string, string> map = new Dictionary<string, string>();
-         foreach (ColumnInfo c in r.schema.columnInfo) {
-            map.Add(c.colName, r[c.colName].ToString());
-         }
-         return map;
-      }
-
-      public static string to_json(this IBlueRow r) {
-         Dictionary<string, string> map = r.to_dict();
-         string json = System.Text.Json.JsonSerializer.Serialize(map);
-         return json;
-      }
-
       public static void compute_differences_fast(IBlueRowCollection new_rows, IBlueRowCollection old_rows,
                                                   out IBlueRowCollection out_d_rows, out IBlueRowCollection out_a_rows) {
          if (old_rows == null || old_rows.Count == 0) {

src/DataStreamSuite/DataStreamCore/DataStreamCore.csproj

@@ -2,7 +2,7 @@
 
   <PropertyGroup>
     <AllowUnsafeBlocks>true</AllowUnsafeBlocks>
-    <TargetFramework>net8.0</TargetFramework>
+    <TargetFramework>net7.0</TargetFramework>
     <Nullable>disable</Nullable>
   </PropertyGroup>
 

src/DataStreamSuite/DataStreamCore/DynamicLoading/PrimitiveManager.cs

@@ -47,10 +47,10 @@ namespace BCG {
       /// </summary>
       /// <param name="assemblyName"></param>
       public void loadAssembly(string assemblyName) {
-         Func<Type, bool> is_blue_orb = (t) => {
-            //if (!assemblyName.Contains("SharedStreams.dll"))
-            //   return true;
-            return t.Namespace == "bts.BlueOrb";// || t.Namespace == "BCG";
+         Func<Type, bool> is_silverpine_only = (t) => {
+            if (!assemblyName.Contains("SharedStreams.dll"))
+               return true;
+            return t.Namespace == "bts.SilverPine";
          };
          bts.Log.Gen.info($"PrimitiveManager.loadAssembly assemblyName: {assemblyName}");
          if (myAssemblyNames.Contains(assemblyName))
@@ -61,7 +61,7 @@ namespace BCG {
             Assembly.LoadFrom(assemblyName);
 
          foreach (Type type in loadedAssembly.GetTypes())
-            if (!type.IsInterface && is_blue_orb(type)
+            if (!type.IsInterface && is_silverpine_only(type)
                && !type.IsAbstract
                && type.IsSubclassOf(typeof(ViewStream))
                && type.IsPublic) {
@@ -77,7 +77,7 @@ namespace BCG {
                      break;
                   }
                if (isVisible) {
-                 // Log.assert_d(myTypeLookupTable[userRep] == null, type.FullName);
+                  Log.assert_d(myTypeLookupTable[userRep] == null, type.FullName);
                   myTypeLookupTable[userRep] = type;
                }
             }

src/DataStreamSuite/DataStreamExpressions/DataStreamExpressions.csproj

@@ -1,7 +1,7 @@
 <Project Sdk="Microsoft.NET.Sdk">
 
   <PropertyGroup>
-    <TargetFramework>net8.0</TargetFramework>
+    <TargetFramework>net7.0</TargetFramework>
     <ImplicitUsings>enable</ImplicitUsings>
     <Nullable>enable</Nullable>
   </PropertyGroup>

src/DataStreamSuite/DataStreamPublisher/DataStreamPublisher.csproj

@@ -2,7 +2,7 @@
 
   <PropertyGroup>
     <OutputType>Exe</OutputType>
-    <TargetFramework>net8.0</TargetFramework>
+    <TargetFramework>net7.0</TargetFramework>
     <Nullable>enable</Nullable>
     <AllowUnsafeBlocks>true</AllowUnsafeBlocks>
   </PropertyGroup>

src/DataStreamSuite/DataStreamQueryAnalyzer/DataStreamQueryAnalyzer.csproj

@@ -2,7 +2,7 @@
 
   <PropertyGroup>
     <OutputType>Exe</OutputType>
-    <TargetFramework>net8.0-windows</TargetFramework>
+    <TargetFramework>net7.0-windows</TargetFramework>
     <Nullable>enable</Nullable>
     <UseWindowsForms>true</UseWindowsForms>
   </PropertyGroup>

src/DataStreamSuite/DataStreamTests/DataStreamTests.csproj

@@ -1,7 +1,7 @@
 <Project Sdk="Microsoft.NET.Sdk">
 
   <PropertyGroup>
-    <TargetFramework>net8.0</TargetFramework>
+    <TargetFramework>net7.0</TargetFramework>
     <ImplicitUsings>enable</ImplicitUsings>
     <Nullable>enable</Nullable>
 

src/DataStreamSuite/ExpressionCore/ExpressionCore.csproj

@@ -1,7 +1,7 @@
 <Project Sdk="Microsoft.NET.Sdk">
 
   <PropertyGroup>
-    <TargetFramework>net8.0</TargetFramework>
+    <TargetFramework>net7.0</TargetFramework>
     <ImplicitUsings>enable</ImplicitUsings>
     <Nullable>enable</Nullable>
   </PropertyGroup>

src/DataStreamSuite/GuiUtilities/GuiUtilities.csproj

@@ -1,7 +1,7 @@
 <Project Sdk="Microsoft.NET.Sdk">
 
   <PropertyGroup>
-    <TargetFramework>net8.0-windows</TargetFramework>
+    <TargetFramework>net7.0-windows</TargetFramework>
     <ImplicitUsings>enable</ImplicitUsings>
     <Nullable>enable</Nullable>
     <UseWindowsForms>true</UseWindowsForms>

src/DataStreamSuite/PubSub/Client/PublishingUtilities/IPublisher.cs

@@ -6,18 +6,12 @@ namespace BCG
 	public interface IPublisher
 	{
       void publish( IPubItem pi );
-#pragma warning disable SYSLIB0011 // Type or member is obsolete
       void publish( IPubItem pi, IFormatter f );
-#pragma warning restore SYSLIB0011 // Type or member is obsolete
       void publish( IPubItem[] pi );
-#pragma warning disable SYSLIB0011 // Type or member is obsolete
       void publish( IPubItem[] pi, IFormatter f );
-#pragma warning restore SYSLIB0011 // Type or member is obsolete
    }
 
-#pragma warning disable SYSLIB0011 // Type or member is obsolete
    public interface IPublisher<T> where T : IFormatter
-#pragma warning restore SYSLIB0011 // Type or member is obsolete
    {
       void publish (IPubItem pi);
       void publish (IPubItem[] pi);

src/DataStreamSuite/PubSub/Misc/GenericFormatter.cs

@@ -5,21 +5,23 @@ using System.Runtime.Serialization.Formatters.Binary;
 
 namespace BCG
 {
-#pragma warning disable SYSLIB0011 // Type or member is obsolete
    public class GenericFormatter<F> : IGenericFormatter where F : IFormatter, new ()
    {
       IFormatter myFormatter = new F ();
 
       public T Deserialize<T> (Stream serializationStream)
       {
+         #pragma warning disable 
          return (T) myFormatter.Deserialize (serializationStream);
+         #pragma warning restore SYSLIB0011
       }
       public void Serialize<T> (Stream serializationStream, T graph)
       {
+         #pragma warning disable 
          myFormatter.Serialize (serializationStream, graph);
+         #pragma warning restore SYSLIB0011
       }
    }
 
    public class GenericBinaryFormatter : GenericFormatter<BinaryFormatter> { }
-#pragma warning restore SYSLIB0011 // Type or member is obsolete
 }

src/DataStreamSuite/PubSub/PubSub.csproj

@@ -6,7 +6,7 @@
 
   <PropertyGroup>
       <AllowUnsafeBlocks>true</AllowUnsafeBlocks>
-    <TargetFramework>net8.0</TargetFramework>
+    <TargetFramework>net7.0</TargetFramework>
     <Nullable>enable</Nullable>
   </PropertyGroup>
 

src/DataStreamSuite/PubSubServer/PubSubServer.csproj

@@ -2,7 +2,7 @@
 
   <PropertyGroup>
     <OutputType>Exe</OutputType>
-    <TargetFramework>net8.0</TargetFramework>
+    <TargetFramework>net7.0</TargetFramework>
     <Nullable>enable</Nullable>
   </PropertyGroup>
 

src/DataStreamSuite/SharedStreams/Business/MarketData/Analytics/SpatialArbScoreStream.cs

@@ -1,18 +0,0 @@
-namespace bts.BlueOrb {
-   using bts.SilverPine;
-   using BCG;
-   using System;
-
-   [DataStreamUserRespresentation("SpatialArbScoreStream")]
-   public class SpatialArbScoreStream : ViewStream {
-      public SpatialArbScoreStream(int window_sz_secs, int sample_period_sz)
-         : this(TimeSpan.FromSeconds(window_sz_secs), TimeSpan.FromSeconds(sample_period_sz), ViewStreamDatabase.instance) { }
-
-      public SpatialArbScoreStream(TimeSpan window_sz, TimeSpan sample_period, ViewStreamDatabase vsd) {
-         ViewStream rs = new SpatialArbStatsStream(window_sz, sample_period, vsd);
-         rs = new ThrottleStream(rs, TimeSpan.FromSeconds(1), vsd);
-         rs = new SortStream(rs, "-n_trade_ticks");
-         mySetRealSource(rs);
-      }
-   }
-}

src/DataStreamSuite/SharedStreams/Business/MarketData/Analytics/SpatialArbStatsStream.cs

@@ -1,113 +0,0 @@
-namespace bts.SilverPine {
-   using BCG;
-
-   using SharpCompress.Common.Rar;
-
-   using System;
-   using System.Collections.Generic;
-   using System.Linq;
-   using System.Text;
-   using System.Threading.Tasks;
-
-   // For all the pairs generated by SpatialFuturePairsStream, this stream 
-   // generates the statistics, by appending together a SpreadStatsStream
-   [DataStreamUserRespresentation("SpatialArbStatsStream")]
-   public class SpatialArbStatsStream : ViewStream{
-      private ViewStream _spatial_pairs_stream;
-      private ViewStreamDatabase _vsd;
-      private ViewStream _tot_str = null;
-      private HashSet<string> _key_map = new HashSet<string>();
-      private ViewStream _bbo_stream;
-      private readonly TimeSpan _window_sz, _sampling_period_sz;
-
-      public SpatialArbStatsStream(int window_sz_seconds, int sampling_period_sz_seconds)
-         : this(TimeSpan.FromSeconds(window_sz_seconds), TimeSpan.FromSeconds(sampling_period_sz_seconds), ViewStreamDatabase.instance) { }
-
-      public SpatialArbStatsStream(TimeSpan window_sz, TimeSpan sampling_period_sz, ViewStreamDatabase vsd) {
-         _vsd = vsd;
-         _window_sz = window_sz;
-         _sampling_period_sz = sampling_period_sz;
-         _spatial_pairs_stream = new SpatialFuturePairsStream(vsd);
-         _bbo_stream = new BBOStream(vsd);
-      }
-
-      protected override void myOnFirstSubscriber() {
-         _spatial_pairs_stream.onChange += _spatial_pairs_stream_onchange;
-      }
-
-      protected override void myOnLastSubscriber() {
-         _spatial_pairs_stream.onChange -= _spatial_pairs_stream_onchange;
-      }
-
-      private ViewStream _build_mapped_stream(ViewStream parent_str) {
-         Func<IBlueTableSchema, IBlueTableSchema> build_schema = (inp_schema) => {
-            IBlueTableSchema rv = new BlueTableSchema("SpreaddStats", new ColumnInfo("id", 0, TypeCode.Int32));
-            rv = rv.add_column(TypeCode.String, "spread");
-            rv = rv.add_column(TypeCode.String, "window");
-            // bids
-            rv = rv.add_column(TypeCode.Double, "bid_mean");
-            rv = rv.add_column(TypeCode.Double, "bid_stddev");
-            rv = rv.add_column(TypeCode.Double, "bid_skewness");
-            rv = rv.add_column(TypeCode.Double, "bid_kurtosis");
-            rv = rv.add_column(TypeCode.Double, "bid_max");
-            rv = rv.add_column(TypeCode.Double, "bid_min");
-            // asks
-            rv = rv.add_column(TypeCode.Double, "ask_mean");
-            rv = rv.add_column(TypeCode.Double, "ask_stddev");
-            rv = rv.add_column(TypeCode.Double, "ask_skewness");
-            rv = rv.add_column(TypeCode.Double, "ask_kurtosis");
-            rv = rv.add_column(TypeCode.Double, "ask_max");
-            rv = rv.add_column(TypeCode.Double, "ask_min");
-            // tail distribution
-            rv = rv.add_column(TypeCode.Double, "bid Z ≥ 2σ");
-            rv = rv.add_column(TypeCode.Double, "ask Z < -2σ");
-            // trading ticks
-            rv = rv.add_column(TypeCode.Int32, "n_buy_ticks");
-            rv = rv.add_column(TypeCode.Double, "cumm_buy_price");
-            rv = rv.add_column(TypeCode.Int32, "n_sell_ticks");
-            rv = rv.add_column(TypeCode.Double, "cumm_sell_price");
-            rv = rv.add_column(TypeCode.Int32, "n_trade_ticks");
-            return rv;
-         };
-         Func<IBlueRow, IBlueRow> build_row = (r) => {
-            IBlueTableSchema s = build_schema(r.schema);
-            return new BlueRow(s, r["id"], r["spread"], r["window"],
-                           r["bid_mean"], r["bid_stddev"], r["bid_skewness"], r["bid_kurtosis"], r["bid_max"], r["bid_min"],
-                           r["ask_mean"], r["ask_stddev"], r["ask_skewness"], r["ask_kurtosis"], r["ask_max"], r["ask_min"],
-                           r["bid Z ≥ 2σ"], r["ask Z < -2σ"],
-                           r["n_buy_ticks"], r["cumm_buy_price"], r["n_sell_ticks"], r["cumm_sell_price"], r["n_trade_ticks"]);
-         };
-         MapStream rs = new MapStream(parent_str, inp_schema => build_schema(inp_schema),
-            inp_r => build_row(inp_r));
-         return rs;
-      }
-
-      private void _spatial_pairs_stream_onchange(object dp_obj) {
-         if (_tot_str != null)
-            return;
-         MutableAppendStream append_str = new MutableAppendStream();
-         _tot_str = _build_mapped_stream(append_str);
-         //_tot_str = new ThrottleStream(_tot_str, 1000);
-         //_tot_str = new SortStream(_tot_str, "-n_trade_ticks");
-         _tot_str.onChange += _handle_tot_stream;
-
-         DataPacket dp = dp_obj as DataPacket;         
-         foreach (IBlueRow r in dp.newRows) {
-            string s1 = r.get<string>("s1");
-            string s2 = r.get<string>("s2");
-            string key = SpatialFuturePairsStream.get_key(s1, s2);
-            if (!_key_map.Contains(key)) {
-               _key_map.Add(key);
-               ViewStream spread_stats_stream = new SpreadStatsStream(s1, s2, 1, 1, Convert.ToInt32(_window_sz.TotalSeconds), 
-                                                                     Convert.ToInt32(_sampling_period_sz.TotalSeconds), _bbo_stream, _vsd);
-               append_str.addStream(spread_stats_stream);
-            }
-         }
-      }
-
-      private void _handle_tot_stream(object dp_obj) {
-         myFireEvent(dp_obj);
-      }
-   }
-
-}

src/DataStreamSuite/SharedStreams/Business/MarketData/Analytics/SpatialFuturePairsStream.cs

@@ -1,89 +0,0 @@
-namespace bts.SilverPine {
-   using BCG;
-
-   using bts.api;
-
-   using System;
-   using System.Collections.Generic;
-   using System.Linq;
-   using System.Text;
-   using System.Threading.Tasks;
-
-   [DataStreamUserRespresentation("SpatialFuturePairsStream")]
-   public class SpatialFuturePairsStream : ViewStream {
-      private ViewStream _main_stream;
-      private IBlueTableSchema _schema;
-      private HashSet<string> _key_map = new HashSet<string>();
-      public SpatialFuturePairsStream()
-         : this(ViewStreamDatabase.instance) { }
-
-      public SpatialFuturePairsStream(ViewStreamDatabase vsd) {
-         _main_stream = new InstrumentsStream(vsd);
-         _main_stream = new FilterStream(_main_stream, "sec_type='Future'");
-         _schema = new BlueTableSchema("SpatialFutures", new ColumnInfo("id", 0, TypeCode.Int32));
-         _schema = _schema.add_column(TypeCode.String, "s1");
-         _schema = _schema.add_column(TypeCode.String, "venue_1");
-         _schema = _schema.add_column(TypeCode.String, "s2");
-         _schema = _schema.add_column(TypeCode.String, "venue_2");
-
-      }
-
-      protected override void myOnFirstSubscriber() {
-         _main_stream.onChange += _main_stream_onchange;
-      }
-
-      protected override void myOnLastSubscriber() {
-         _main_stream.onChange -= _main_stream_onchange;
-      }
-
-      public static string get_key(string s1, string s2) {
-         string key = string.Compare(s1, s2, StringComparison.Ordinal) <= 0 ? s1 + s2 : s2 + s1;
-         return key;
-      }
-
-      public static string get_key(IBlueRow r1, IBlueRow r2) {
-         string s1 = r1.get<string>("sec");
-         string s2 = r2.get<string>("sec");
-         return get_key(s1, s2);
-      }
-
-      private bool _combine(IBlueRow r1, IBlueRow r2) {
-         string key = get_key(r1, r2);
-         if (_key_map.Contains(key))
-            return false;
-         if (r1.get<string>("symbol") != r2.get<string>("symbol"))
-            return false;
-         if (r1.get_enum<Venue>("venue") == r2.get_enum<Venue>("venue"))
-            return false;
-         if (r1.get<int>("sec_key") == r2.get<int>("sec_key")) {
-            return false;
-         }
-         DateTime e1 = r1.get_datetime("expiration");
-         DateTime e2 = r2.get_datetime("expiration");
-         bool exps_eq = e1 == e2 || (e1.Year >= 2100 && e2.Year >= 2100);
-         if (!exps_eq)
-            return false;         
-         return true;
-      }
-
-      private void _main_stream_onchange(object dp_obj) {
-         _key_map.Clear();
-         DataPacket dp = dp_obj as DataPacket;
-         BlueTable table = new BlueTable(_schema);
-         var rows = dp.newRows.to_list();
-         foreach (var r1 in rows) {
-            foreach (var r2 in rows) {
-               if (_combine(r1, r2)) {
-                  IBlueRow nr = new BlueRow(_schema, table.Count, r1.get<string>("sec"), r1.get<string>("venue"),
-                     r2.get<string>("sec"), r2.get<string>("venue"));
-                  table.addRow(nr);
-                  string key = get_key(r1, r2);
-                  _key_map.Add(key);
-               }
-            }
-         }
-         var dp_out = BlueDataUtility.build_dp(table, myLastValue as DataPacket);
-         myFireEvent(dp_out);
-      }
-   }
-}

src/DataStreamSuite/SharedStreams/Business/MarketData/Analytics/SpreadPerformanceEvaluator.cs

@@ -1,656 +0,0 @@
-using System;
-using System.Collections.Generic;
-using System.Linq;
-using MathNet.Numerics.Statistics;
-using MathNet.Numerics.LinearRegression;
-using System.Collections.Generic;
-using System.Linq;
-using MathNet.Numerics.Statistics;
-using MathNet.Numerics.LinearAlgebra;
-using Amazon.Runtime;
-using System;
-using System.Collections.Generic;
-using System.Linq;
-using MathNet.Numerics.Statistics;
-using System;
-using System.Collections.Generic;
-using System.Linq;
-using MathNet.Numerics.Statistics;
-using System;
-using System.Collections.Generic;
-using System.Linq;
-using MathNet.Numerics.Statistics;
-// stats calculator
-using System;
-using System.Collections.Generic;
-using System.Linq;
-using MathNet.Numerics.Statistics;
-
-/// <summary>
-/// Provides evaluation metrics to assess the quality of a synthetic spread for mean-reversion trading.
-/// Assumes the input spread data is a trailing, time-causal window of recent ticks (e.g., last 120 seconds).
-/// </summary>
-public static class SpreadPerformanceEvaluator
-{
-
-   public static class SpreadStatistics {
-      public class SpreadStatsResult {
-         public Dictionary<string, double> bid_zscore_distribution { get; set; }
-         public Dictionary<string, double> ask_zscore_distribution { get; set; }
-         public Dictionary<string, double> mid_zscore_distribution { get; set; }
-
-         public DescriptiveStatistics bid_stats { get; set; }
-         public DescriptiveStatistics ask_stats { get; set; }
-         public DescriptiveStatistics mid_stats { get; set; }
-      }
-
-      private static readonly string[] _zscore_labels = new[]
-      {
-        "Z < -2σ",
-        "-2σ ≤ Z < -1σ",
-        "-1σ ≤ Z < 0σ",
-        "0σ ≤ Z < 1σ",
-        "1σ ≤ Z < 2σ",
-        "Z ≥ 2σ"
-    };
-
-      public static SpreadStatsResult analyze_spread_quotes(List<(DateTime timestamp, double bid, double bid_qty, double ask, double ask_qty)> quotes) {
-         if (quotes == null || quotes.Count == 0)
-            throw new ArgumentException("Spread quote list cannot be null or empty.");
-
-         var bid_prices = quotes.Select(q => q.bid).ToList();
-         var ask_prices = quotes.Select(q => q.ask).ToList();
-         var mid_prices = quotes.Select(q => (q.bid + q.ask) / 2.0).ToList();
-
-         var bid_stats = new DescriptiveStatistics(bid_prices);
-         var ask_stats = new DescriptiveStatistics(ask_prices);
-         var mid_stats = new DescriptiveStatistics(mid_prices);
-
-         var bid_zscores = bid_prices.Select(x => zscore(x, bid_stats.Mean, bid_stats.StandardDeviation)).ToList();
-         var ask_zscores = ask_prices.Select(x => zscore(x, ask_stats.Mean, ask_stats.StandardDeviation)).ToList();
-         var mid_zscores = mid_prices.Select(x => zscore(x, mid_stats.Mean, mid_stats.StandardDeviation)).ToList();
-
-         var bid_distribution = bin_zscores_to_percentages(bid_zscores);
-         var ask_distribution = bin_zscores_to_percentages(ask_zscores);
-         var mid_distribution = bin_zscores_to_percentages(mid_zscores);
-
-         return new SpreadStatsResult {
-            bid_zscore_distribution = bid_distribution,
-            ask_zscore_distribution = ask_distribution,
-            mid_zscore_distribution = mid_distribution,
-            bid_stats = bid_stats,
-            ask_stats = ask_stats,
-            mid_stats = mid_stats
-         };
-      }
-
-      private static double zscore(double x, double mean, double std_dev) {
-         return std_dev == 0 ? 0 : (x - mean) / std_dev;
-      }
-
-      private static Dictionary<string, double> bin_zscores_to_percentages(List<double> zscores) {
-         var bins = _zscore_labels.ToDictionary(label => label, _ => 0);
-
-         foreach (var z in zscores) {
-            if (z < -2) bins["Z < -2σ"]++;
-            else if (z < -1) bins["-2σ ≤ Z < -1σ"]++;
-            else if (z < 0) bins["-1σ ≤ Z < 0σ"]++;
-            else if (z < 1) bins["0σ ≤ Z < 1σ"]++;
-            else if (z < 2) bins["1σ ≤ Z < 2σ"]++;
-            else bins["Z ≥ 2σ"]++;
-         }
-
-         double total = zscores.Count;
-         return bins.ToDictionary(kvp => kvp.Key, kvp => total == 0 ? 0 : (kvp.Value / total) * 100.0);
-      }
-
-      public static double score_spread(SpreadStatsResult result) {
-         var ask_bins = result.ask_zscore_distribution;
-         var bid_bins = result.bid_zscore_distribution;
-         var mid_bins = result.mid_zscore_distribution;
-
-         double buy_opportunity = ask_bins["Z < -2σ"] + ask_bins["-2σ ≤ Z < -1σ"];
-         double sell_opportunity = bid_bins["Z ≥ 2σ"] + bid_bins["1σ ≤ Z < 2σ"];
-         double opportunity_score = buy_opportunity + sell_opportunity;
-
-         double below_mean = mid_bins["Z < -2σ"] + mid_bins["-2σ ≤ Z < -1σ"] + mid_bins["-1σ ≤ Z < 0σ"];
-         double above_mean = mid_bins["Z ≥ 2σ"] + mid_bins["1σ ≤ Z < 2σ"] + mid_bins["0σ ≤ Z < 1σ"];
-         double symmetry_score = 1.0 - Math.Abs(below_mean - above_mean) / (below_mean + above_mean + 1e-6);
-
-         double amplitude_score = result.mid_stats.StandardDeviation;
-         double skew_penalty = 1.0 / (1.0 + Math.Abs(result.mid_stats.Skewness));
-         double kurtosis_bonus = Math.Min(result.mid_stats.Kurtosis, 6.0) / 6.0;
-
-         double score =
-             opportunity_score *
-             symmetry_score *
-             amplitude_score *
-             skew_penalty *
-             (1.0 + kurtosis_bonus * 0.1);
-
-         return score;
-      }
-   }
-
-
-
-
-
-/// <summary>
-/// Represents a single trade action (entry or exit) taken during the simulation.
-/// </summary>
-public struct TradeAction
-    {
-        public bool IsEntry { get; set; }    // True if this action opened a position
-        public bool IsLong { get; set; }     // True = Long, False = Short
-        public double Price { get; set; }    // Executed price
-        public int Index { get; set; }       // Tick index of the action
-
-        public override string ToString()
-        {
-            string side = IsLong ? "Long" : "Short";
-            string type = IsEntry ? "Open" : "Close";
-            return $"{type} {side} @ {Price:F4} (Tick {Index})";
-        }
-    }
-
-    /// <summary>
-/// Simulates a mean-reversion strategy using rolling, causal mid-price stats.
-/// ✅ Entry:
-///   - Buy at ask when ask < rollingMean - entryThreshold × rollingStdDev
-///   - Sell at bid when bid > rollingMean + entryThreshold × rollingStdDev
-/// ✅ Exit:
-///   - Close when mid returns within rollingMean ± exitThreshold × rollingStdDev
-///   - Long exits at bid, short exits at ask
-/// This version uses only past and current data at each step (no forward-looking bias).
-/// </summary>
-public static (double TotalPnL, List<TradeAction> TradeActions, int BuyTurns, int SellTurns) EvaluateSimulatedPnL(
-    List<Tuple<double, double, double, double>> spreadData,
-    int statWindow = 30,
-    double entryThresholdMultiplier = 0.5,
-    double exitThresholdMultiplier = 0.1)
-{
-    var trades = new List<TradeAction>();
-    var mids = spreadData.Select(x => (x.Item1 + x.Item3) / 2.0).ToList();
-
-    if (spreadData.Count < statWindow)
-        return (0.0, trades, 0, 0);
-
-    bool positionOpen = false;
-    bool isLong = false;
-    double entryPrice = 0.0;
-    double pnl = 0.0;
-    int buyTurns = 0;
-    int sellTurns = 0;
-
-    for (int i = statWindow; i < spreadData.Count; i++)
-    {
-        double bid = spreadData[i].Item1;
-        double ask = spreadData[i].Item3;
-        double mid = mids[i];
-
-        // Rolling window
-        var window = mids.Skip(i - statWindow).Take(statWindow).ToList();
-        double mean = window.Average();
-        double stddev = Math.Sqrt(window.Select(x => Math.Pow(x - mean, 2)).Average());
-
-        double lowerThreshold = mean - entryThresholdMultiplier * stddev;
-        double upperThreshold = mean + entryThresholdMultiplier * stddev;
-        double exitBand = exitThresholdMultiplier * stddev;
-
-        if (!positionOpen)
-        {
-            if (ask < lowerThreshold)
-            {
-                positionOpen = true;
-                isLong = true;
-                entryPrice = ask;
-                trades.Add(new TradeAction { IsEntry = true, IsLong = true, Price = entryPrice, Index = i });
-            }
-            else if (bid > upperThreshold)
-            {
-                positionOpen = true;
-                isLong = false;
-                entryPrice = bid;
-                trades.Add(new TradeAction { IsEntry = true, IsLong = false, Price = entryPrice, Index = i });
-            }
-        }
-        else
-        {
-            if (Math.Abs(mid - mean) < exitBand)
-            {
-                double exitPrice = isLong ? bid : ask;
-                pnl += isLong ? (exitPrice - entryPrice) : (entryPrice - exitPrice);
-
-                trades.Add(new TradeAction { IsEntry = false, IsLong = isLong, Price = exitPrice, Index = i });
-
-                if (isLong) buyTurns++;
-                else sellTurns++;
-
-                positionOpen = false;
-            }
-        }
-    }
-
-    return (pnl, trades, buyTurns, sellTurns);
-}
-
-
-
-    /// <summary>
-    /// Estimates the Hurst exponent of the spread mid-prices.
-    /// Assumes the input is a trailing window of recent data.
-    /// Hurst < 0.5 implies mean-reversion; ideal is 0.35–0.45.
-    /// </summary>
-    public static double EvaluateHurstExponent(List<Tuple<double, double, double, double>> spreadData)
-    {
-        const int MinLength = 21;
-
-        if (spreadData == null || spreadData.Count < MinLength)
-            return double.NaN;
-
-        var spreadMids = spreadData.Select(x => (x.Item1 + x.Item3) / 2.0).ToArray();
-        int n = spreadMids.Length;
-
-        var lags = new List<double>();
-        var tau = new List<double>();
-
-        for (int lag = 2; lag <= 20; lag++)
-        {
-            if (n - lag <= 1)
-                continue;
-
-            double[] diffs = new double[n - lag];
-            for (int i = 0; i < diffs.Length; i++)
-                diffs[i] = spreadMids[i + lag] - spreadMids[i];
-
-            double stddev = diffs.StandardDeviation();
-
-            if (stddev > 0)
-            {
-                lags.Add(Math.Log(lag));
-                tau.Add(Math.Log(stddev));
-            }
-        }
-
-        if (lags.Count < 2)
-            return double.NaN;
-
-        var (intercept, slope) = SimpleRegression.Fit(lags.ToArray(), tau.ToArray());
-        double hurst = slope * 0.5 + 0.5;
-        return hurst;
-    }
-
-    /// <summary>
-/// Computes the mean and standard deviation of the spread mid-prices
-/// over a trailing, causal window.
-/// </summary>
-/// <param name="spreadData">List of (bid, bidQty, ask, askQty)</param>
-/// <returns>Tuple of (Mean, StdDev). Returns (NaN, NaN) if input is too short.</returns>
-public static (double Mean, double StdDev) EvaluateMeanAndStdDev(List<Tuple<double, double, double, double>> spreadData)
-{
-    if (spreadData == null || spreadData.Count < 2)
-        return (double.NaN, double.NaN);
-
-    var mids = spreadData.Select(x => (x.Item1 + x.Item3) / 2.0).ToList();
-    double mean = mids.Average();
-    double stddev = Math.Sqrt(mids.Select(x => Math.Pow(x - mean, 2)).Average());
-
-    return (mean, stddev);
-}
-
-
-   /// <summary>
-   /// Counts how many times the spread ask dropped below (mean - threshold) [buy signal]
-   /// and how many times the spread bid rose above (mean + threshold) [sell signal].
-   /// Assumes spreadData is a trailing, causal window.
-   /// </summary>
-   /// <returns>(BuyExcursions, SellExcursions)</returns>
-   public static (uint BuyExcursions, double cumm_buy_price, uint SellExcursions, double cumm_sell_price) EvaluateExcursionCount(
-       List<Tuple<double, double, double, double>> spreadData,
-       double entryThresholdMultiplier = 0.5) {
-      if (spreadData == null || spreadData.Count < 2)
-         return (0, Double.NaN, 0, Double.NaN);
-
-      var midPrices = spreadData.Select(x => (x.Item1 + x.Item3) / 2.0).ToList();
-      double mean = midPrices.Average();
-      double stddev = midPrices.StandardDeviation();
-      double buy_threshold = mean - entryThresholdMultiplier * stddev;
-      double sell_threshold = mean + entryThresholdMultiplier * stddev;
-
-      int buyCount = 0;
-      int sellCount = 0;
-      //bool in_buy_zone = false;
-      //bool in_sell_zone = false;
-
-      double cumm_buy_price = 0.0, cumm_sell_price = 0.0;
-      foreach (var (bid, _, ask, _) in spreadData) {
-         if (ask <= buy_threshold) {
-            //if (!in_buy_zone) {
-            cumm_buy_price += ask;
-            buyCount++;
-            //in_buy_zone = true;
-         }
-         //} else {
-         //   in_buy_zone = false;
-         //}
-
-         if (bid >= sell_threshold) {
-            //if (!in_sell_zone) {
-            cumm_sell_price += bid;
-            sellCount++;
-            //in_sell_zone = true;
-         }
-         //} else {
-         //   in_sell_zone = false;
-         //}
-      }
-
-      return ((uint)buyCount, cumm_buy_price / buyCount, (uint)sellCount, cumm_sell_price / sellCount);
-   }
-
-/// <summary>
-/// Counts how many times the spread mid-price crosses its mean level.
-/// Each crossing indicates a potential oscillation turn (cycle midpoint).
-/// Assumes the input is a trailing, causal window of spread ticks.
-/// </summary>
-/// <param name="spreadData">List of (bid, bidQty, ask, askQty)</param>
-/// <returns>Number of zero crossings in the mid-price series</returns>
-public static UInt32 EvaluateZeroCrossingCount(List<Tuple<double, double, double, double>> spreadData)
-{
-    if (spreadData == null || spreadData.Count < 2)
-        return 0;
-
-    var mids = spreadData.Select(x => (x.Item1 + x.Item3) / 2.0).ToList();
-    double mean = mids.Average();
-    int zeroCrossings = 0;
-
-    for (int i = 1; i < mids.Count; i++)
-    {
-        double prevOffset = mids[i - 1] - mean;
-        double currOffset = mids[i] - mean;
-
-        if (prevOffset * currOffset < 0) // Sign changed
-            zeroCrossings++;
-    }
-
-    return (UInt32)zeroCrossings;
-}
-
-   /// <summary>
-   /// Computes the half-life of mean reversion.
-   /// Lower half-life means the spread reverts to its mean more quickly.
-   /// </summary>
-   public static double ComputeHalfLife(List<Tuple<double, double, double, double>> spreadData) {
-      var prices = spreadData.Select(x => (x.Item1 + x.Item3) / 2.0).ToList();
-      var lagged = prices.Take(prices.Count - 1).ToArray();
-      var delta = prices.Skip(1).Zip(lagged, (cur, prev) => cur - prev).ToArray();
-
-      if (lagged.Length < 2) return double.NaN;
-
-      var x = Vector<double>.Build.Dense(lagged);
-      var y = Vector<double>.Build.Dense(delta);
-      var beta = x.DotProduct(y) / x.DotProduct(x);
-
-      return -Math.Log(2) / beta;
-   }
-
-   /// <summary>
-   /// Computes the lag-1 autocorrelation of the spread.
-   /// Lower values indicate quicker mean-reversion.
-   /// </summary>
-   public static double ComputeLag1Autocorrelation(List<Tuple<double, double, double, double>> spreadData) {
-      var prices = spreadData.Select(x => (x.Item1 + x.Item3) / 2.0).ToList();
-      var mean = prices.Average();
-      var numerator = 0.0;
-      var denominator = 0.0;
-
-      for (int i = 1; i < prices.Count; i++) {
-         numerator += (prices[i] - mean) * (prices[i - 1] - mean);
-         denominator += Math.Pow(prices[i - 1] - mean, 2);
-      }
-
-      return denominator == 0 ? 0 : numerator / denominator;
-   }
-
-   /// <summary>
-   /// Estimates how quickly extreme Z-score values decay back toward the mean.
-   /// Shorter average decay time indicates stronger mean-reversion.
-   /// </summary>
-   public static double ComputeAverageZScoreDecay(List<Tuple<double, double, double, double>> spreadData, int zScoreWindow = 50) {
-      var prices = spreadData.Select(x => (x.Item1 + x.Item3) / 2.0).ToList();
-      if (prices.Count < zScoreWindow + 2) return double.NaN;
-
-      var decayTimes = new List<int>();
-      for (int i = zScoreWindow; i < prices.Count - 1; i++) {
-         var window = prices.Skip(i - zScoreWindow).Take(zScoreWindow).ToList();
-         double mean = window.Average();
-         double stdDev = Statistics.StandardDeviation(window);
-         if (stdDev == 0) continue;
-
-         double z = (prices[i] - mean) / stdDev;
-         if (Math.Abs(z) > 2) {
-            for (int j = 1; j < prices.Count - i; j++) {
-               double zNext = (prices[i + j] - mean) / stdDev;
-               if (Math.Abs(zNext) < 1) {
-                  decayTimes.Add(j);
-                  break;
-               }
-            }
-         }
-      }
-
-      return decayTimes.Count == 0 ? double.NaN : decayTimes.Average();
-   }
-
-   /// <summary>
-   /// Computes the variance ratio: short-term variance divided by long-term variance.
-   /// Lower ratio suggests stronger mean-reversion.
-   /// </summary>
-   public static double ComputeVarianceRatio(List<Tuple<double, double, double, double>> spreadData, int shortWindow, int longWindow) {
-      var prices = spreadData.Select(x => (x.Item1 + x.Item3) / 2.0).ToList();
-      if (prices.Count < longWindow) return double.NaN;
-
-      double shortVar = Statistics.Variance(prices.Skip(prices.Count - shortWindow).Take(shortWindow));
-      double longVar = Statistics.Variance(prices.Skip(prices.Count - longWindow).Take(longWindow));
-
-      return longVar == 0 ? double.NaN : shortVar / longVar;
-   }
-
-   /// <summary>
-   /// Computes the highest bid and the lowest ask from a spread tick series.
-   /// Useful for evaluating the best market opportunity range.
-   /// </summary>
-   /// <param name="spreadData">List of (bid, bidQty, ask, askQty)</param>
-   /// <returns>Tuple of (MaxBid, MinAsk)</returns>
-   public static (double MaxBid, double MinAsk) EvaluateMaxBidMinAsk(List<Tuple<double, double, double, double>> spreadData)
-{
-    if (spreadData == null || spreadData.Count == 0)
-        return (double.NaN, double.NaN);
-
-    double maxBid = double.NegativeInfinity;
-    double minAsk = double.PositiveInfinity;
-
-    foreach (var (bid, _, ask, _) in spreadData)
-    {
-        if (bid > maxBid) maxBid = bid;
-        if (ask < minAsk) minAsk = ask;
-    }
-
-    return (maxBid, minAsk);
-}
-
-/// <summary>
-/// Computes the distribution of spread mid-prices around the mean,
-/// grouped by caller-defined distance bands (in standard deviations).
-///
-/// This helps answer:
-/// - Is the spread tightly hugging the mean?
-/// - Are there wide excursions or heavy tails?
-///
-/// You define breakpoints like: [0.5, 1.0, 2.0] → it will return:
-/// - "≤ 0.5σ"
-/// - "0.5σ – 1.0σ"
-/// - "1.0σ – 2.0σ"
-/// - "> 2.0σ"
-///
-/// How to Interpret It:
-/// - ≤ 0.5σ        → Tight clustering near the mean (strong mean-reversion)
-/// - 0.5σ – 1.0σ   → Mild oscillation (common trade region)
-/// - 1.0σ – 2.0σ   → Moderate deviation (potential trade signals)
-/// - > 2.0σ        → Rare, extreme moves (tail events, potential entries/exits)
-///
-/// Use this to tune entry thresholds or detect regime shifts.
-/// </summary>
-/// <param name="spreadData">List of (bid, bidQty, ask, askQty)</param>
-/// <param name="regions">List of positive breakpoints in std deviations (e.g., [0.5, 1.0, 2.0])</param>
-/// <returns>Dictionary mapping label to % of mid-prices in that region</returns>
-public static Dictionary<string, double> EvaluateDistributionAroundMean(
-    List<Tuple<double, double, double, double>> spreadData,
-    List<double> regions)
-{
-    var result = new Dictionary<string, double>();
-
-    if (spreadData == null || spreadData.Count < 2 || regions == null || regions.Count == 0)
-        return result;
-
-    var mids = spreadData.Select(x => (x.Item1 + x.Item3) / 2.0).ToList();
-    double mean = mids.Average();
-    double stddev = Math.Sqrt(mids.Select(x => Math.Pow(x - mean, 2)).Average());
-
-    if (stddev == 0)
-    {
-        result["Zero Volatility"] = 100.0;
-        return result;
-    }
-
-    var buckets = new int[regions.Count + 1];
-    foreach (var mid in mids)
-    {
-        double distance = Math.Abs(mid - mean);
-        bool placed = false;
-
-        for (int i = 0; i < regions.Count; i++)
-        {
-            if (distance <= regions[i])
-            {
-                buckets[i]++;
-                placed = true;
-                break;
-            }
-        }
-
-        if (!placed)
-            buckets[regions.Count]++;
-    }
-
-    int total = mids.Count;
-    for (int i = 0; i < regions.Count; i++)
-    {
-        string label = i == 0
-            ? $"≤ {regions[i]:0.##}σ"
-            : $"{regions[i - 1]:0.##}σ – {regions[i]:0.##}σ";
-        result[label] = 100.0 * buckets[i] / total;
-    }
-
-    result[$"> {regions.Last():0.##}σ"] = 100.0 * buckets.Last() / total;
-
-    return result;
-}
-
-/// <summary>
-/// Computes two drift metrics indicating how much the spread's mean changes over time,
-/// based on the standard deviation of rolling mean values.
-///
-/// ✅ Absolute Drift: Standard deviation of the rolling mean
-/// ✅ Relative Drift: Drift normalized by the overall mean (unitless ratio)
-///
-/// This shows whether the center of the spread is stable — critical for mean-reversion.
-///
-/// How to Interpret the Output:
-/// - ✅ Relative Drift < 1% → Very stable mean — ideal
-/// - ⚠️ 1–5% → Mild drift — consider shorter holds or adaptive mean
-/// - ❌ >5% → Drifting mean — fixed mean may no longer work
-/// </summary>
-/// <param name="spreadData">List of (bid, bidQty, ask, askQty)</param>
-/// <param name="window">Window size for rolling mean (e.g., 20 ticks)</param>
-/// <returns>
-/// Tuple of (Absolute Drift, Relative Drift) — where:
-/// - Absolute Drift = StdDev of rolling means
-/// - Relative Drift = Drift / global mean
-/// </returns>
-public static (double AbsoluteDrift, double RelativeDrift) EvaluateMeanDriftScore(
-    List<Tuple<double, double, double, double>> spreadData,
-    int window = 20)
-{
-    if (spreadData == null || spreadData.Count < window)
-        return (double.NaN, double.NaN);
-
-    var mids = spreadData.Select(x => (x.Item1 + x.Item3) / 2.0).ToArray();
-    var rollingMeans = new List<double>();
-
-    for (int i = window; i <= mids.Length; i++)
-    {
-        double rollingMean = mids.Skip(i - window).Take(window).Average();
-        rollingMeans.Add(rollingMean);
-    }
-
-
-    double overallMean = mids.Average(); // use full mean for normalization
-    //double drift = Math.Sqrt(rollingMeans.Select(m => Math.Pow(m - rollingMeans.Average(), 2)).Average());
-    double drift = Math.Sqrt(rollingMeans.Select(m => Math.Pow(m - rollingMeans.Average(), 2)).Average());
-
-
-double relativeDrift = Math.Abs(overallMean) > 1e-8 ? drift / Math.Abs(overallMean) : double.NaN;
-    return (drift, relativeDrift);
-}
-
-/// <summary>
-/// Returns the latest Kalman Filter estimate of the true mean of the spread mid-prices.
-/// This tracks the spread's central value while allowing for gradual drift and suppressing noise.
-///
-/// Internally, this applies a recursive 1D Kalman filter over the input mid-prices.
-/// 
-/// Tuning Parameters:
-/// - Q: Process noise variance (higher = faster adaptation to drift)
-/// - R: Measurement noise variance (higher = more smoothing)
-/// 
-/// Use this instead of SMA if your mean drifts or if price series is noisy.
-///
-/// </summary>
-/// <param name="spreadData">List of (bid, bidQty, ask, askQty)</param>
-/// <param name="Q">Process noise variance (default = 0.0001)</param>
-/// <param name="R">Measurement noise variance (default = 0.01)</param>
-/// <returns>Latest estimated Kalman-filtered mean, or NaN if input is invalid</returns>
-public static double GetLatestKalmanMean(
-    List<Tuple<double, double, double, double>> spreadData,
-    double Q = 0.0001,
-    double R = 0.01)
-{
-    if (spreadData == null || spreadData.Count == 0)
-        return double.NaN;
-
-    var mids = spreadData.Select(x => (x.Item1 + x.Item3) / 2.0).ToList();
-
-    double x = mids[0];  // initial estimate of mean
-    double P = 1.0;       // initial uncertainty
-
-    for (int i = 1; i < mids.Count; i++)
-    {
-        // Predict step
-        double x_prior = x;
-        double P_prior = P + Q;
-
-        // Update step
-        double K = P_prior / (P_prior + R);
-        x = x_prior + K * (mids[i] - x_prior);
-        P = (1 - K) * P_prior;
-    }
-
-    return x; // the latest estimated mean
-}
-
-
-
-
-}

src/DataStreamSuite/SharedStreams/Business/MarketData/Analytics/SpreadPerformanceStream.cs

@@ -1,137 +0,0 @@
-namespace bts.SilverPine {
-   using BCG;
-
-   using System.Diagnostics;
-
-   public class SpreadPerformanceStream : ViewStream {
-      ViewStream _spread_price_stream = null;
-      private readonly IBlueTableSchema _schema;
-      private readonly int _window_secs;
-      private readonly double _buy_threshold, _sell_threshold, _flat_threshold;
-      private ViewStream _bbo_stream;
-      public SpreadPerformanceStream(string s1, string s2, double w1, double w2, int window_secs,
-                                       double buy_threshold, double sell_threshold, double flat_threshold)
-      : this(s1, s2, w1, w2, window_secs, buy_threshold, sell_threshold, flat_threshold, ViewStreamDatabase.instance) { }
-
-      public SpreadPerformanceStream(string s1, string s2, double w1, double w2, int window_secs,
-                                      double buy_threshold, double sell_threshold, double flat_threshold,
-                                      ViewStreamDatabase vsd) {
-         _bbo_stream = new BBOStream(vsd);
-         _schema = BlueDataUtility.form_schema("SpreadPerformance", Tuple.Create("id", TypeCode.Int32),                                                
-                                                Tuple.Create("t0", TypeCode.DateTime),
-                                                Tuple.Create("t1", TypeCode.DateTime),
-                                                Tuple.Create("n_ticks", TypeCode.UInt32),
-                                                Tuple.Create("curr_windows_seconds", TypeCode.Double),
-                                                // means / std dev                                          
-                                                Tuple.Create("stddev", TypeCode.Double),
-                                                Tuple.Create("mean", TypeCode.Double),
-                                                Tuple.Create("kalman_filter_mean", TypeCode.Double),
-                                                Tuple.Create("absolute_mean_drift", TypeCode.Double),
-                                                Tuple.Create("relative_mean_drift", TypeCode.Double),
-                                                // statistical measures
-                                                Tuple.Create("hurst_exponent", TypeCode.Double),
-                                                Tuple.Create("half_life", TypeCode.Double),
-                                                Tuple.Create("lag1_autocorrelation", TypeCode.Double),
-                                                Tuple.Create("average_zscore_decay", TypeCode.Double),
-                                                Tuple.Create("variance_ratio", TypeCode.Double),
-                                                Tuple.Create("n_zero_crossings", TypeCode.UInt32),                                                
-                                                // distributions
-                                                Tuple.Create("Z ≤ 0.5σ", TypeCode.Double),
-                                                Tuple.Create("0.5σ – 1.0σ", TypeCode.Double),
-                                                Tuple.Create("1.0σ – 2.0σ", TypeCode.Double),
-                                                Tuple.Create("> 2.0σ", TypeCode.Double),
-                                                ////////////////////////////////////////                                                   
-                                                Tuple.Create("max_bid", TypeCode.Double),
-                                                Tuple.Create("min_ask", TypeCode.Double),
-                                                Tuple.Create("n_buy_signals", TypeCode.UInt32),
-                                                Tuple.Create("cumm_buy_price", TypeCode.Double),
-                                                Tuple.Create("n_sell_signals", TypeCode.UInt32),
-                                                Tuple.Create("cumm_sell_price", TypeCode.Double)
-                                                //Tuple.Create("pnl", TypeCode.Double),
-                                                //Tuple.Create("buy_turns", TypeCode.UInt32),
-                                                //Tuple.Create("sell_turns", TypeCode.UInt32),
-                                                //Tuple.Create("trades", TypeCode.String)
-                                                );
-         _window_secs = window_secs;
-         _buy_threshold = buy_threshold;
-         _sell_threshold = sell_threshold;
-         _flat_threshold = flat_threshold;
-         _spread_price_stream = new SyntheticSpreadStream(s1, s2, w1, w2, _bbo_stream, vsd);
-         _spread_price_stream = new TimeSeriesStream(_spread_price_stream);
-         _spread_price_stream = new ThrottleStream(_spread_price_stream, TimeSpan.FromSeconds(5.0), vsd);
-      }
-
-      protected override void myOnFirstSubscriber() {
-         _spread_price_stream.onChange += _handle_spread_prices;
-      }
-
-      protected override void myOnLastSubscriber() {
-         _spread_price_stream.onChange -= _handle_spread_prices;
-      }
-
-      private void _handle_spread_prices(object dp_obj) {
-         DataPacket dp = dp_obj as DataPacket;
-         List<Tuple<DateTime, double, double, double, double>> spread_ts_prices = new List<Tuple<DateTime, double, double, double, double>>();
-         DateTime min_time = DateTime.MaxValue; DateTime max_time = DateTime.MinValue;
-         var sw = Stopwatch.StartNew();
-         sw.Start();
-         foreach (IBlueRow r in dp.newRows) {
-            DateTime ts = r.get_datetime("timestamp");
-            TimeSpan elapsed_time = DateTime.Now - ts;
-            if (elapsed_time.TotalSeconds >= _window_secs)
-               continue;
-            spread_ts_prices.Add(Tuple.Create(
-                                          r.get_datetime("timestamp"),
-                                          r.get<double>("spread_bid_price"),
-                                          r.get<double>("spread_bid_sz"),
-                                          r.get<double>("spread_ask_price"),
-                                          r.get<double>("spread_ask_sz")));
-            if (ts <= min_time) {
-               min_time = ts;
-            }
-            if (ts >= max_time) {
-               max_time = ts;
-            }
-         }
-         List<Tuple<double, double, double, double>> spread_prices = spread_ts_prices
-                                                                                             .OrderBy(t => t.Item1)
-                                                                                             .Select(t => Tuple.Create(t.Item2, t.Item3, t.Item4, t.Item5))
-                                                                                             .ToList();
-
-         uint n_ticks = (uint)spread_prices.Count;
-         double curr_windows_seconds = (max_time - min_time).TotalSeconds;
-         double hurst_exponent = SpreadPerformanceEvaluator.EvaluateHurstExponent(spread_prices);
-         double half_life = SpreadPerformanceEvaluator.ComputeHalfLife (spread_prices);
-         double lag1_autocorrelation = SpreadPerformanceEvaluator.ComputeLag1Autocorrelation(spread_prices);
-         double average_zscore_decay = SpreadPerformanceEvaluator.ComputeAverageZScoreDecay(spread_prices);
-         double variance_ratio = SpreadPerformanceEvaluator.ComputeVarianceRatio(spread_prices, 60, 360);
-
-         (double mean, double stddev) = SpreadPerformanceEvaluator.EvaluateMeanAndStdDev(spread_prices);
-         double kalman_filter_mean = SpreadPerformanceEvaluator.GetLatestKalmanMean(spread_prices);
-         uint n_zero_crossings = SpreadPerformanceEvaluator.EvaluateZeroCrossingCount(spread_prices);
-         (double max_bid, double min_ask) = SpreadPerformanceEvaluator.EvaluateMaxBidMinAsk(spread_prices);
-         (uint n_buy_signals, double cumm_buy_price, uint n_sell_signals, double cumm_sell_price) = SpreadPerformanceEvaluator.EvaluateExcursionCount(spread_prices, _buy_threshold);
-         //(double pnl, List<SpreadPerformanceEvaluator.TradeAction> trades, int buy_turns, int sell_turns) = SpreadPerformanceEvaluator.EvaluateSimulatedPnL(spread_prices, 30, _buy_threshold, _flat_threshold);
-         //string trade_str = String.Empty;
-         //foreach (var tr in trades) {
-         //   trade_str += tr.ToString() + " , ";
-         //}
-         Dictionary<string, double> dist_map = SpreadPerformanceEvaluator.EvaluateDistributionAroundMean(spread_prices,
-                                                                       new List<double>(new double[] { 0.5, 1.0, 2.0 }));
-         var dist_vals = dist_map.Count < 4 ? new double[] { Double.NaN, Double.NaN, Double.NaN, Double.NaN } : dist_map.Values.ToArray();
-         (double absolute_drift, double relative_drift) = SpreadPerformanceEvaluator.EvaluateMeanDriftScore(spread_prices);
-         IBlueRow nr = new BlueRow(_schema, 0, min_time, max_time, n_ticks, curr_windows_seconds, stddev, mean, kalman_filter_mean, absolute_drift, relative_drift,
-            hurst_exponent, half_life, lag1_autocorrelation, average_zscore_decay, variance_ratio, n_zero_crossings,
-                                    dist_vals[0], dist_vals[1], dist_vals[2], dist_vals[3],
-                                    max_bid, min_ask, n_buy_signals, cumm_buy_price, n_sell_signals, cumm_sell_price);
-         BlueTable t = new BlueTable(_schema);
-         t.addRow(nr);
-         var dp_out = BlueDataUtility.build_dp_inc(t, myLastValue as DataPacket, true);
-         var perf_msecs = sw.ElapsedMilliseconds;
-         sw.Stop();
-         Log.Gen.info($"SpreadPerformanceStream._handle_spread_prices comp_time: {perf_msecs} msecs");
-         myFireEvent(dp_out);
-
-      }
-   }
-}

src/DataStreamSuite/SharedStreams/Business/MarketData/Analytics/SpreadStatArbsWeightLinearRegressionStream.cs

@@ -1,68 +0,0 @@
-namespace bts.SilverPine{
-   using BCG;
-   public class SpreadStatArbsWeightLinearRegressionStream : ViewStream {
-      private IBlueTableSchema _schema;
-      private ViewStream _leg_price_stream;
-      private readonly double _window_secs;
-      private readonly string _s1, _s2;
-
-      public SpreadStatArbsWeightLinearRegressionStream(string s1, string s2, double window_secs, double sample_period_secs)
-         : this(s1, s2, window_secs, sample_period_secs, ViewStreamDatabase.instance) { }
-
-
-      public SpreadStatArbsWeightLinearRegressionStream(string s1, string s2, double window_secs, double sample_period_secs, ViewStreamDatabase vsd) {
-         _window_secs = window_secs;
-         _s1 = s1;
-         _s2 = s2;
-         _schema = BlueDataUtility.form_schema("LinearRegressionWeights",
-                                                Tuple.Create("id", TypeCode.Int32),
-                                                Tuple.Create("t0", TypeCode.DateTime),
-                                                Tuple.Create("t1", TypeCode.DateTime),
-                                                Tuple.Create("curr_windows_seconds", TypeCode.Double),
-                                                Tuple.Create("n_ticks", TypeCode.UInt32),
-                                                Tuple.Create("s1", TypeCode.String),
-                                                Tuple.Create("w1", TypeCode.Double),
-                                                Tuple.Create("s2", TypeCode.String),
-                                                Tuple.Create("w2", TypeCode.String));
-
-         _leg_price_stream = new TimeSeriesStream(new SyntheticSpreadLegsStream(s1, s2));
-         _leg_price_stream = new ThrottleStream(_leg_price_stream, TimeSpan.FromSeconds(1), vsd);
-      }
-
-      protected override void myOnFirstSubscriber() {
-         _leg_price_stream.onChange += _handle_leg_prices;
-      }
-
-      protected override void myOnLastSubscriber() {
-         _leg_price_stream.onChange -= _handle_leg_prices;
-      }
-
-      private void _handle_leg_prices(object dp_obj){
-         DataPacket dp = dp_obj as DataPacket;
-         List<Tuple<double, double, DateTime>> leg_prices = new List<Tuple<double, double, DateTime>>();
-         foreach (IBlueRow r in dp.newRows){
-            double s1_mid = (r.get<double>("s1_bid_price") + r.get<double>("s1_ask_price")) /2.0;
-            double s2_mid = (r.get<double>("s2_bid_price") + r.get<double>("s2_ask_price")) /2.0;
-            DateTime ts = r.get_datetime("timestamp");
-            leg_prices.Add(new Tuple<double, double, DateTime>(s1_mid, s2_mid, ts));
-         }
-         if (leg_prices.Count <= 2)
-            return;
-         leg_prices = leg_prices.Where(t => (DateTime.Now - t.Item3).TotalSeconds <= _window_secs).ToList();
-         DateTime t0 = leg_prices.Select(t => t.Item3).Min();
-         DateTime t1 = leg_prices.Select(t => t.Item3).Max();
-         double curr_windows_seconds = (t1 - t0).TotalSeconds;
-         int n_ticks = leg_prices.Count;
-         List<double> s1_mids = leg_prices.Select(t => t.Item1).ToList();
-         List<double> s2_mids = leg_prices.Select(t => t.Item2).ToList();
-         (double w1, double w2) = SpreadWeightCalculator.CalculateWeightsLinearRegression(s1_mids, s2_mids);
-         BlueTable state = new BlueTable(_schema);
-         IBlueRow nr = new BlueRow(_schema, 0, t0, t1, curr_windows_seconds, n_ticks,_s1, w1, _s2, w2);
-         state.addRow(nr);
-         var dp_out = BlueDataUtility.build_dp_inc(state, myLastValue as DataPacket, true);
-         myFireEvent(dp_out);
-
-      }
-   }
-}
-

src/DataStreamSuite/SharedStreams/Business/MarketData/Analytics/SpreadStatsStream.cs

@@ -1,143 +0,0 @@
-namespace bts.SilverPine {
-   using BCG;
-
-   using MathNet.Numerics.LinearAlgebra.Factorization;
-
-   using System;
-   using System.Collections.Generic;
-   using System.Diagnostics;
-   using System.Linq;
-   using System.Text;
-   using System.Threading.Tasks;
-   [DataStreamUserRespresentation("SpreadStatsStream")]
-   // This is the fundamental stream that computes the stats (including buy and sell zones) 
-   // of a spread. the legs and the weights are passed as args
-   public class SpreadStatsStream : ViewStream {
-      private ViewStream _spread_price_stream;
-      private readonly IBlueTableSchema _schema;
-      private readonly TimeSpan _window_sz;
-      private readonly string _spread;
-      public SpreadStatsStream(string s1, string s2, double w1, double w2, int rolling_window_sz_seconds, int window_sampling_sz_secs)
-         : this(s1, s2, w1, w2, rolling_window_sz_seconds, window_sampling_sz_secs, new BBOStream(ViewStreamDatabase.instance), ViewStreamDatabase.instance) { }
-      public SpreadStatsStream(string s1, string s2, double w1, double w2, int rolling_window_sz_seconds, int window_sampling_sz_secs, ViewStream bbo_stream, ViewStreamDatabase vsd) {
-         _window_sz = TimeSpan.FromSeconds(rolling_window_sz_seconds);
-         _spread = $"{w1} X {s1} - {w2} X {s2}";
-         _schema = new BlueTableSchema("SpreaddStats", new ColumnInfo ("id", 0, TypeCode.Int32));
-         _schema = _schema.add_column(TypeCode.String, "spread");
-         _schema = _schema.add_column(TypeCode.String, "window");
-         // basic stats
-         _schema = _schema.add_column(TypeCode.Double, "bid_mean");
-         _schema = _schema.add_column(TypeCode.Double, "ask_mean");
-         _schema = _schema.add_column(TypeCode.Double, "bid_stddev");
-         _schema = _schema.add_column(TypeCode.Double, "ask_stddev");
-         _schema = _schema.add_column(TypeCode.Double, "bid_skewness");
-         _schema = _schema.add_column(TypeCode.Double, "ask_skewness");
-         _schema = _schema.add_column(TypeCode.Double, "bid_kurtosis");
-         _schema = _schema.add_column(TypeCode.Double, "ask_kurtosis");
-         _schema = _schema.add_column(TypeCode.Double, "bid_max");
-         _schema = _schema.add_column(TypeCode.Double, "ask_max");
-         _schema = _schema.add_column(TypeCode.Double, "bid_min");
-         _schema = _schema.add_column(TypeCode.Double, "ask_min");
-         // bid bins
-         _schema = _schema.add_column(TypeCode.Double, "bid Z < -2σ");
-         _schema = _schema.add_column(TypeCode.Double, "bid -2σ ≤ Z < -1σ");
-         _schema = _schema.add_column(TypeCode.Double, "bid -1σ ≤ Z < 0σ");
-         // positive z's
-         _schema = _schema.add_column(TypeCode.Double, "bid 0σ ≤ Z < 1σ");
-         _schema = _schema.add_column(TypeCode.Double, "bid 1σ ≤ Z < 2σ");
-         _schema = _schema.add_column(TypeCode.Double, "bid Z ≥ 2σ");
-         // ask bins
-         _schema = _schema.add_column(TypeCode.Double, "ask Z < -2σ");
-         _schema = _schema.add_column(TypeCode.Double, "ask -2σ ≤ Z < -1σ");
-         _schema = _schema.add_column(TypeCode.Double, "ask -1σ ≤ Z < 0σ");
-         // positive z's
-         _schema = _schema.add_column(TypeCode.Double, "ask 0σ ≤ Z < 1σ");
-         _schema = _schema.add_column(TypeCode.Double, "ask 1σ ≤ Z < 2σ");
-         _schema = _schema.add_column(TypeCode.Double, "ask Z ≥ 2σ");
-         // opps
-         _schema = _schema.add_column(TypeCode.Int32, "n_buy_ticks");
-         _schema = _schema.add_column(TypeCode.Double, "cumm_buy_price");
-         _schema = _schema.add_column(TypeCode.Int32, "n_sell_ticks");
-         _schema = _schema.add_column(TypeCode.Double, "cumm_sell_price");
-         _schema = _schema.add_column(TypeCode.Int32, "n_trade_ticks");
-         _spread_price_stream = new SyntheticSpreadStream(s1, s2, w1, w2, bbo_stream, vsd);
-         _spread_price_stream = new TimeSeriesStream(_spread_price_stream);
-         double window_sampling_sz_smsecs = window_sampling_sz_secs * 1000.0 + new Random(DateTime.Now.Second).Next(-100, 100);
-         TimeSpan sampling_period = TimeSpan.FromMilliseconds(window_sampling_sz_smsecs);
-         _spread_price_stream = new ThrottleStream(_spread_price_stream, sampling_period, vsd);
-      }
-
-      protected override void myOnFirstSubscriber() {
-         _spread_price_stream.onChange += _handle_spread_prices;
-      }
-
-      protected override void myOnLastSubscriber() {
-         _spread_price_stream.onChange -= _handle_spread_prices;
-      }
-
-      private void _handle_spread_prices(object dp_obj) {
-         List<IBlueRow> rows = (dp_obj as DataPacket).newRows.to_list();
-         Func<IBlueRow, bool> row_filter = (r) => {
-            DateTime ts = r.get_datetime("timestamp");
-            DateTime start_time = DateTime.Now.Subtract(_window_sz);
-            return ts >= start_time;
-         };
-         var rolling_win_rows = rows.Where(r => row_filter(r));
-         var spread_prices = rolling_win_rows.Select(r => (
-                                          r.get_datetime("timestamp"),
-                                          r.get<double>("spread_bid_price"),
-                                          r.get<double>("spread_bid_sz"),
-                                          r.get<double>("spread_ask_price"),
-                                          r.get<double>("spread_ask_sz"))).ToList();
-         spread_prices.Sort((a,b) => a.Item1.CompareTo(b.Item1));
-         DateTime t0 = spread_prices.First().Item1, t1 = spread_prices.Last().Item1;
-         var tot_secs = (t1 - t0).TotalSeconds;
-         var stopwatch = Stopwatch.StartNew();
-         Log.Gen.info($"SpreadStatsStream.analyze_spread_quotes executed in {stopwatch.ElapsedMilliseconds} ms");
-         var result = SpreadPerformanceEvaluator.SpreadStatistics.analyze_spread_quotes(spread_prices);
-         (uint n_buy_signals, double cumm_buy_price, uint n_sell_signals, double cumm_sell_price) = 
-              SpreadPerformanceEvaluator.EvaluateExcursionCount(spread_prices.Select(t => Tuple.Create(t.Item2, t.Item3, t.Item4, t.Item5)).ToList(), 2.0);
-         IBlueRow nr = new BlueRow(_schema,
-            0,
-            _spread,
-            $"n_ticks: {spread_prices.Count} t0: {t0.ToLongTimeString()} -> t1: {t1.ToLongTimeString()} ({tot_secs}) secs",
-            result.bid_stats.Mean,
-            result.ask_stats.Mean,
-            result.bid_stats.StandardDeviation,
-            result.ask_stats.StandardDeviation,
-            result.bid_stats.Skewness,
-            result.ask_stats.Skewness,
-            result.bid_stats.Kurtosis,
-            result.ask_stats.Kurtosis,
-            result.bid_stats.Maximum,
-            result.ask_stats.Maximum,
-            result.bid_stats.Minimum,
-            result.ask_stats.Minimum,
-            // bid distribution
-            result.bid_zscore_distribution["Z < -2σ"],
-            result.bid_zscore_distribution["-2σ ≤ Z < -1σ"],
-            result.bid_zscore_distribution["-1σ ≤ Z < 0σ"],
-            result.bid_zscore_distribution["0σ ≤ Z < 1σ"],
-            result.bid_zscore_distribution["1σ ≤ Z < 2σ"],
-            result.bid_zscore_distribution["Z ≥ 2σ"],
-            // ask distributions
-            result.ask_zscore_distribution["Z < -2σ"],
-            result.ask_zscore_distribution["-2σ ≤ Z < -1σ"],
-            result.ask_zscore_distribution["-1σ ≤ Z < 0σ"],
-            result.ask_zscore_distribution["0σ ≤ Z < 1σ"],
-            result.ask_zscore_distribution["1σ ≤ Z < 2σ"],
-            result.ask_zscore_distribution["Z ≥ 2σ"],
-            n_buy_signals,
-            cumm_buy_price,
-            n_sell_signals,
-            cumm_sell_price,
-            n_buy_signals + n_sell_signals
-            );
-         stopwatch.Stop();
-         BlueTable t = new BlueTable(_schema);
-         t.addRow(nr);
-         var dp_out = BlueDataUtility.build_dp(t, myLastValue as DataPacket);
-         myFireEvent(dp_out);
-      }
-   }
-}

src/DataStreamSuite/SharedStreams/Business/MarketData/Analytics/SpreadWeightCalculator.cs

@@ -1,49 +0,0 @@
-using System;
-using System.Collections.Generic;
-using MathNet.Numerics.LinearRegression;
-
-namespace bts.SilverPine {
-
-   public static class SpreadWeightCalculator {
-      /// <summary>
-      /// Estimates hedge weights (w1, w2) such that the spread s = w1 * s1 - w2 * s2 
-      /// is potentially mean-reverting. Uses linear regression: s1 = beta * s2 + ε.
-      /// 
-      /// Assumes mid-prices are precomputed and aligned over the same time window.
-      /// </summary>
-      /// <param name="s1MidPrices">List of mid-prices for asset 1 (dependent variable)</param>
-      /// <param name="s2MidPrices">List of mid-prices for asset 2 (independent variable)</param>
-      /// <param name="includeIntercept">
-      /// Whether to include an intercept in the regression model (default: true).
-      /// </param>
-      /// <returns>(w1, w2): Tuple of hedge weights</returns>
-      public static (double w1, double w2) CalculateWeightsLinearRegression(
-          List<double> s1MidPrices,
-          List<double> s2MidPrices,
-          bool includeIntercept = true) {
-         if (s1MidPrices == null || s2MidPrices == null)
-            throw new ArgumentNullException("Price series cannot be null.");
-         if (s1MidPrices.Count != s2MidPrices.Count)
-            throw new ArgumentException("Price series must be of equal length.");
-         if (s1MidPrices.Count < 2)
-            throw new ArgumentException("Need at least two data points for regression.");
-
-         double[] x = s2MidPrices.ToArray(); // Independent variable
-         double[] y = s1MidPrices.ToArray(); // Dependent variable
-
-         double beta;
-
-         if (includeIntercept) {
-            // Returns (intercept, slope)
-            (double intercept, double slope) = SimpleRegression.Fit(x, y);
-            beta = slope;
-         } else {
-            // Returns only slope when forcing regression through origin
-            beta = SimpleRegression.FitThroughOrigin(x, y);
-         }
-
-         // Return fixed w1 = 1.0 and w2 = beta
-         return (1.0, beta);
-      }
-   }
-}

src/DataStreamSuite/SharedStreams/Business/MarketData/Analytics/VWAPStream.cs

@@ -1,118 +0,0 @@
-// TODO: Remove
-//// namespace bts.SilverPine 
-////    using bts;
-////    using BCG;
-////    using MongoDB.Driver.Linq;
-////    using System.ComponentModel.DataAnnotations;
-////    using ZstdSharp.Unsafe;
-
-////    [DataStreamUserRespresentation("VWAP")]
-////    public class VWAPStream : ViewStream {
-////       // returns the VWAP stream for each contract 
-////       private static ViewStream build_vwap_stream(string symbol, DateTime begin_dt, DateTime end_dt) {
-////          var ohlcv_params = OHLCVStream.get_params(begin_dt, end_dt);
-////          ViewStream ohlc_str = new OHLCVStream(symbol, ohlcv_params);
-////          // stream is grouped per contract and then vwap is calculated per contract
-////          GlobalMapStream rs = new GlobalMapStream(ohlc_str, "ts_event", "contract",
-////          (rows, idx) => {
-////             Log.Gen.info($"VWAPStream.GlobalMapStream delegate n_rows: {rows.Count} idx: {idx}");
-////             IBlueRow ir = rows.ElementAt((int)idx);
-////             var sel_rows = rows.Skip((int)idx).Where(r => r.get_datetime("ts_event") >= end_dt && r.get_datetime("ts_event") <= begin_dt);
-////             var volumeprices = sel_rows.Select(r => Tuple.Create(r.get<int>("volume"), r.get<double>("close"))).ToList();
-////             double vwap = IndicatorCalc.calculate_VWAP(volumeprices);
-////             IBlueTableSchema schema = ir.schema.add_columns(Tuple.Create(typeof(double), "vwap"));
-////             MutableBlueRow nr = new MutableBlueRow(schema, ir);
-////             //vwap = -313.987;
-////             nr.setValue("vwap", vwap);
-////             return nr;
-////          });
-////          return rs;
-////       }
-////       // the args want dt1 > dt2, if its nto we, swap them
-////       static void swap_if_necessary(ref DateTime dt1, ref DateTime dt2) {
-////          if (dt1 < dt2) {
-////             DateTime aux = dt1;
-////             dt1 = dt2;
-////             dt2 = aux;
-////          }
-////       }
-
-////       // this assumes the ohlc_str has diff contracts
-////       public VWAPStream(string symbol_or_contract, DateTime begin_dt, DateTime end_dt) {
-////          swap_if_necessary(ref begin_dt, ref end_dt);
-////          Tuple<ContractUtils.ContractCategory, string> ccat_sym_contr = ContractUtils.get_symbol_category(symbol_or_contract);
-////          string symbol = String.Empty;
-////          symbol = ccat_sym_contr.Item2;
-////          var ohlcv_params = OHLCVStream.get_params(begin_dt, end_dt);
-////          ViewStream vwap_str = build_vwap_stream(symbol, begin_dt, end_dt);
-////          // vwap_str now has vwap for all contracts. if the arg was a symbol, we need to filter it for the active contract,
-////          // whihc we do with FutureActivesContractStream. If the arg was a contract we'll simply filter it
-////          ViewStream rs = vwap_str;
-////          switch (ccat_sym_contr.Item1) {
-////             case ContractUtils.ContractCategory.FutureString:
-////             case ContractUtils.ContractCategory.SpreadString:
-////                rs = new FilterStream(vwap_str, r => r.get<string>("contract").CompareTo(ccat_sym_contr.Item2) == 0);
-////                break;
-////             case ContractUtils.ContractCategory.Symbol:
-////                rs = new FuturesActiveContractStream(vwap_str);
-////                break;
-////          }
-////          rs = new FilterStream(rs, r => r.get_datetime("ts_event").CompareTo(begin_dt) == 0);
-////          rs = new NewProjectStream(rs, "! vwap");
-////          mySetRealSource(rs);
-////       }
-
-////       public VWAPStream(string symbol_or_contract, string begin_dt, string end_dt)
-////       :
-////       this(symbol_or_contract, DateTime.Parse(begin_dt), DateTime.Parse(end_dt)) { }
-////    }
-//// }
-
-//using System;
-
-//using BCG;
-
-//using bts.api;
-
-//namespace bts.SilverPine {
-//   /*
-//      ----------------------------------------------------------------------
-//                                  *Expression*
-//      ----------------------------------------------------------------------
-//          //nl("Find Settlement price of the SP500 future in the first week of 2023.")
-
-//          r=FutureHistoricalBars("ES", "One_Day", "1/1/2024", "2/1/2024")
-//          r=math(r, "vprice = volume * close")
-//          r=aggregate(r, "vprice:Sum, volume:Sum")
-//          r=math(r, "vwap = vprice_sum / volume_sum")
-//          r=newProject(r, "vwap !")
-
-//      ----------------------------------------------------------------------
-//                                    *Schema*
-//      ----------------------------------------------------------------------
-//          id                              System.Int32
-//          vwap                            System.Double
-//   */
-
-//   [DataStreamUserRespresentation("VWAP")]
-//   public class VWAPStream : ViewStream {
-//      public VWAPStream(string symbol_or_contract, DateTime beg_dt, DateTime end_dt)
-//      :
-//      this(symbol_or_contract, beg_dt, end_dt, ViewStreamDatabase.instance) { }
-
-//      public VWAPStream(string symbol_or_contract, DateTime beg_dt, DateTime end_dt, ViewStreamDatabase db) {
-//         //var fc = new ContractUtils.FutureContract(symbol_or_contract);
-//         ViewStream r = new FutureHistoricalBarsStream(symbol_or_contract, beg_dt.ToString(), end_dt.ToString());
-//         // since this is already filtered, we can groupBy iit without a groupBy column
-//         r = new MathStream(r,
-//                 "vprice = volume * close");
-//         r = new NonIncGroupByStream(r,
-//                                      "vprice:Sum, volume:Sum");
-//         r = new MathStream(r,
-//                             "vwap = vprice_sum / volume_sum");
-//         r = new NewProjectStream(r,
-//                                   "vwap !");
-//         mySetRealSource(r);
-//      }
-//   }
-//}

src/DataStreamSuite/SharedStreams/Business/MarketData/BBOStream.cs

@@ -13,6 +13,7 @@ namespace bts.SilverPine {
    using Spark.Api.MarketData;
 
    using SecurityKey = System.UInt32;
+   using System.Data.Entity.Infrastructure;
 
    public class FakeBBOStream : ViewStream, IWorker {
       private string _symbol;
@@ -104,14 +105,14 @@ namespace bts.SilverPine {
    public class BBOStream : ViewStream, IWorker {
       private readonly TimeSpan _poll_period = TimeSpan.FromMilliseconds(500);
       private DateTime _last_poll_time = DateTime.MinValue;
-      private ViewStreamDatabase? _vsd;
-      IReadOnlyDictionary<SecurityKey, BookTuple>? _cached_state = null;
-      MarketDataClient? _md_client;
+      private ViewStreamDatabase _vsd;
+      private int _id = 0;
+      IReadOnlyDictionary<SecurityKey, BookTuple> _cached_state = null;
+      MarketDataClient _md_client;
       internal static IBlueTableSchema _bbo_schema = BlueDataUtility.form_schema("bbo",
                                                 Tuple.Create("id", TypeCode.Int32),
                                                 Tuple.Create("sec_key", TypeCode.UInt32),
                                                 Tuple.Create("sec", TypeCode.String),
-                                                Tuple.Create("exchange", TypeCode.String),
                                                 Tuple.Create("bid_price", TypeCode.Double),
                                                 Tuple.Create("bid_sz", TypeCode.Single),
                                                 Tuple.Create("ask_price", TypeCode.Double),
@@ -123,18 +124,17 @@ namespace bts.SilverPine {
       }
 
       public BBOStream(ViewStreamDatabase vsd) {
-         bts.Log.Gen.info("BBOStream.cons");
-         _vsd = vsd;
-         _md_client = new MarketDataClient(new MDSource(TimeSpan.FromSeconds(0.5)));
-         //FakeBBOStream rs = new FakeBBOStream("Future", "CME", "ES", 5000, 0.25, vsd);
-         //mySetRealSource(rs);
+         // bts.Log.Gen.info("BBOStream.cons");
+         // _vsd = vsd;
+         // _md_client = new MarketDataClient(new MDSource(TimeSpan.FromSeconds(0.5)));
+         FakeBBOStream rs = new FakeBBOStream("Future", "CME", "ES", 5000, 0.25, vsd);
+         mySetRealSource(rs);
       }
 
       protected override void myOnFirstSubscriber() {
          _vsd.addWorker(this);
          _md_client.books().Subscribe(_on_books);
-         _vsd?.addWorker(this);
-         _md_client?.books().Subscribe(_on_books);
+
       }
 
       private void _log_updates(TableUpdate<BookTuple, uint> update) {
@@ -144,24 +144,24 @@ namespace bts.SilverPine {
          }
          foreach (Change<BookTuple> c in update.changes) {
             var bt = c.New;
-            //bts.Log.Gen.info($"BBOStream changes: idx: {i} bid: {bt.book.bids[0].price} / {bt.book.bids[0].size} ask: {bt.book.asks[0].price} / {bt.book.asks[0].size}");
+            bts.Log.Gen.info($"BBOStream changes: idx: {i} bid: {bt.book.bids[0].price} / {bt.book.bids[0].size} ask: {bt.book.asks[0].price} / {bt.book.asks[0].size}");
          }
       }
 
       private void _on_books(TableUpdate<BookTuple, uint> update) {
          _log_updates(update);
          // just cacching in here, will populate the rows in _poll_scan and fire the datapacket
-         //bts.Log.Gen.info($"BBOStream._on_books state: {update.state.Count()} changes: {update.changes.Count()} added: {update.added.Count()}");
+         bts.Log.Gen.info($"BBOStream._on_books state: {update.state.Count()} changes: {update.changes.Count()} added: {update.added.Count()}");
          _cached_state = update.state;
       }
 
       protected override void myOnLastSubscriber() {
-         _vsd?.removeWorker(this);
+         _vsd.removeWorker(this);
       }
 
       private void _scan_bbo() {
          // this shouldn be made incremental using the TableUpdate
-         //bts.Log.Gen.info("BBOStream._scan_bbo");
+         bts.Log.Gen.info("BBOStream._scan_bbo");
          if (_cached_state == null)
             return;
          BlueTable new_state = new BlueTable(_bbo_schema);
@@ -170,23 +170,22 @@ namespace bts.SilverPine {
             var sec = kv.Value.security;
             int id = Convert.ToInt32(skey);
             BookTuple bt = kv.Value;
-            var venue = sec.Exchange;
-            IBlueRow r = new BlueRow(_bbo_schema, id, skey, sec.ToString(), venue.ToString(), bt.book.bids[0].price, bt.book.bids[0].size,
+            IBlueRow r = new BlueRow(_bbo_schema, id, skey, sec.ToString(), bt.book.bids[0].price, bt.book.bids[0].size,
                                      bt.book.asks[0].price, bt.book.asks[0].size);
             new_state.addRow(r);
          }
-         var dp = accumulate(myLastValue as DataPacket, new_state);
+         DataPacket dp = null;
+         if (myLastValue == null) {
+            dp = new DataPacket(new_state, new_state, new BlueTable(_bbo_schema));
+         } else {
+            DataPacket old_dp = myLastValue as DataPacket;
+            IBlueRowCollection added_rows, del_rows;
+            BlueDataUtility.compute_differences_fast(new_state, old_dp.newRows, out del_rows, out added_rows);
+            dp = new DataPacket(new_state, added_rows, del_rows);
+         }
          if (dp.addedRows.Count == 0 && dp.deletedRows.Count == 0)
             return;
          myFireEvent(dp);
-
-         static DataPacket accumulate(DataPacket? old_dp, BlueTable new_state) {
-            if (old_dp is null) {
-               return new(new_state, new_state, new BlueTable(_bbo_schema));
-            }
-            BlueDataUtility.compute_differences_fast(new_state, old_dp.newRows, out var del_rows, out var added_rows);
-            return new(new_state, added_rows, del_rows);
-         }
       }
 
       public void doWork() {
@@ -196,4 +195,4 @@ namespace bts.SilverPine {
          }
       }
    }
-}
+}

src/DataStreamSuite/SharedStreams/Business/MarketData/Bars/HistoricalBarsStream.cs

@@ -1,5 +1,4 @@
-namespace bts.BlueOrb {
-   using bts.SilverPine;
+namespace bts.SilverPine {
    using BCG;
 
    using System.Runtime.CompilerServices;
@@ -46,7 +45,7 @@ namespace bts.BlueOrb {
    }
 
    [DataStreamUserRespresentation("FuturesHistorical1DayBars")]
-   internal class FuturesHistorical1DayBarsStream : ViewStream {
+   public class FuturesHistorical1DayBarsStream : ViewStream {
 
       // this gets the raw bars for a symbol (perhaps make it into a raw bars stream?)
       public FuturesHistorical1DayBarsStream(string symbol) :
@@ -104,7 +103,7 @@ namespace bts.BlueOrb {
    }
 
    [DataStreamUserRespresentation("FuturesHistorical1MinBars")]
-   internal class FuturesHistorical1MinuteBarsStream : ViewStream {
+   public class FuturesHistorical1MinuteBarsStream : ViewStream {
       // same as the above but for one min
       // this gets the raw bars for a symbol
       public FuturesHistorical1MinuteBarsStream(string symbol) :
@@ -188,6 +187,46 @@ namespace bts.BlueOrb {
       }
    }
 
+   [DataStreamUserRespresentation("FuturePriceDiff")]
+   public class FutureHistoricalBarsPriceDiffStream : ViewStream {
+
+      public FutureHistoricalBarsPriceDiffStream(string future_contract, string dt1, string dt2)
+      :
+      this(new ContractUtils.FutureContract(future_contract), DateTime.Parse(dt1), DateTime.Parse(dt2), "close", ViewStreamDatabase.instance) {
+      }
+
+
+      public FutureHistoricalBarsPriceDiffStream(string future_contract, DateTime dt1, DateTime dt2)
+      :
+      this(new ContractUtils.FutureContract(future_contract), dt1, dt2, "close", ViewStreamDatabase.instance) {
+      }
+
+      public FutureHistoricalBarsPriceDiffStream(string future_contract, DateTime dt1, DateTime dt2, string col)
+      :
+      this(new ContractUtils.FutureContract(future_contract), dt1, dt2, col, ViewStreamDatabase.instance) {
+      }
+
+
+      public FutureHistoricalBarsPriceDiffStream(ContractUtils.FutureContract fc, DateTime dt1, DateTime dt2, string col, ViewStreamDatabase vsd) {
+         BarSize bsize = dt2 - dt1 >= TimeSpan.FromDays(1.0) ? BarSize.ONE_DAY : BarSize.ONE_MIN;
+         ViewStream rs = new FutureHistoricalBarsStream(fc, BarSize.ONE_DAY, new DateTimeRange(dt1, dt2));
+
+         rs = new ColumnDifferenceStream(rs, r => {
+            DateTime r_dt = r.get_datetime("ts_event");
+            var rv = dt1.Equals(r_dt);
+            return rv;
+         },
+         //r => dt2.Equals(r.get_datetime("ts_event")), col, col);
+         r => {
+            DateTime r_dt = r.get_datetime("ts_event");
+            var rv = dt2.Equals(r_dt);
+            return rv;
+         }, col, col);
+         rs = new NewProjectStream(rs, "diff !");
+         mySetRealSource(rs);
+      }
+   }
+
    // takes the contract name and expands it into multuiple columns corresponding to the contracts.
    // the contract columns are a union, of all properties with some of them empty
    public class ContractExpanderStream : ViewStream {
@@ -208,29 +247,3 @@ namespace bts.BlueOrb {
    }
 
 }
-
-namespace bts.BlueOrb {
-   using BCG;
-
-   using bts.api;
-
-   using Newtonsoft.Json.Serialization;
-
-   [DataStreamUserRespresentation("FuturesContinousContract")]
-   public class FuturesContinousContractStream : ViewStream {
-      // the argument is symbol, like "ES" etc and not the contract itself
-      public FuturesContinousContractStream(string symbol) : this(symbol, ViewStreamDatabase.instance) { }
-      public FuturesContinousContractStream(string symbol, ViewStreamDatabase db) {
-         symbol = symbol.EndsWith("*") ? symbol : symbol + "*";
-         ViewStream r = new FuturesHistorical1DayBarsStream(symbol);
-         r = new ContractExpanderStream(r, "symbol");
-         r = new FilterStream(r, "contract_type='Future'");
-         // finding the max volume across symbols, if we need to do this for open_interest, then join here 
-         // with the open interest stream on symbol/contract
-         r = new NonIncGroupByStream(r, "ts_event", "volume:Max");
-         r = new NewProjectStream(r, "ts_event symbol !");
-         r = new SortStream(r, "-ts_event");
-         mySetRealSource(r);
-      }
-   }
-}

src/DataStreamSuite/SharedStreams/Business/MarketData/Bars/VolumeAggregationStream.cs

@@ -1,53 +1,46 @@
-//using BCG;
+using BCG;
 
-//namespace bts.SilverPine {
-//   /*
-//      ----------------------------------------------------------------------
-//                                  *Expression*
-//      ----------------------------------------------------------------------
-//          //r=FutureHistoricalBars("ES*", "02/01/2024 09:30 AM", "02/01/2024 09:30 PM")
-//          r=FutureHistoricalBars("ES*","One_Day", 02/01/2024, 02/01/2025)
-//          r=contractExpander(r,"symbol")
-//          r=filter(r,"contract_type='FutureSpread'")
-//          r=aggregate(r,"symbol","volume:Sum","ts_event")
-//          r=aggregate(r,"volume_sum:Max")
-//          r=newProject(r,"volume_sum_max:volume symbol !")
+namespace bts.SilverPine {
+   /*
+      ----------------------------------------------------------------------
+                                  *Expression*
+      ----------------------------------------------------------------------
+          //r=FutureHistoricalBars("ES*", "02/01/2024 09:30 AM", "02/01/2024 09:30 PM")
+          r=FutureHistoricalBars("ES*","One_Day", 02/01/2024, 02/01/2025)
+          r=contractExpander(r,"symbol")
+          r=filter(r,"contract_type='FutureSpread'")
+          r=aggregate(r,"symbol","volume:Sum","ts_event")
+          r=aggregate(r,"volume_sum:Max")
+          r=newProject(r,"volume_sum_max:volume symbol !")
 
-//      ----------------------------------------------------------------------
-//                                    *Schema*
-//      ----------------------------------------------------------------------
-//          symbol                          System.String
-//          id                              System.Int32
-//          volume                          System.Double
-//   */
+      ----------------------------------------------------------------------
+                                    *Schema*
+      ----------------------------------------------------------------------
+          symbol                          System.String
+          id                              System.Int32
+          volume                          System.Double
+   */
 
-//   [DataStreamUserRespresentation("VolumeAgg")]
-//   public class VolumeAggregationStream : ViewStream {
-//      public VolumeAggregationStream(string future_contract, string contract_type, DateTime range_beg, DateTime range_end)
-//      : this(future_contract, contract_type, range_beg, range_end, ViewStreamDatabase.instance) { }
+   [DataStreamUserRespresentation("VolumeAgg")]
+   public class VolumeAggregationStream : ViewStream {
+      public VolumeAggregationStream(string future_contract, string contract_type, DateTime range_beg, DateTime range_end)
+      : this(future_contract, contract_type, range_beg, range_end, ViewStreamDatabase.instance) { }
 
-//      public VolumeAggregationStream(string future_contract, string contract_type, string range_beg, string range_end)
-//      :
-//      this(future_contract, contract_type, DateTime.Parse(range_beg), DateTime.Parse(range_end), ViewStreamDatabase.instance) { }
+      public VolumeAggregationStream(string future_contract, string contract_type, string range_beg, string range_end)
+      :
+      this(future_contract, contract_type, DateTime.Parse(range_beg), DateTime.Parse(range_end), ViewStreamDatabase.instance) { }
 
-//      public VolumeAggregationStream(string future_contract, string contract_type, DateTime range_beg, DateTime range_end, ViewStreamDatabase db) {
-//         future_contract = future_contract.EndsWith("*") ? future_contract : future_contract + "*";
-//         ViewStream r;
-//         r = new FutureHistoricalBarsStream(future_contract,
-//                                             range_beg.ToString(), range_end.ToString());
-//         r = new ContractExpanderStream(r, "symbol");
-//         r = new FilterStream(r, $"contract_type='{contract_type}'");
-//         r = new NonIncGroupByStream(r, "symbol", "volume:Sum", "ts_event");
-//         r = new NonIncGroupByStream(r, "volume_sum:Max");
-//         r = new NewProjectStream(r, "volume_sum_max:volume symbol !");
-//         mySetRealSource(r);
-//      }
-//   }
-
-//   [DataStreamUserRespresentation("PeriodicVolumeAgg")]
-//   public class PeriodicVolumeAggregationStream : ViewStream {
-//      public PeriodicVolumeAggregationStream(string future_contract, string contract_type, DateTime range_beg, DateTime range_end,
-//                                             TimeSpan period, ViewStreamDatabase db) {
-//      }
-//   }
-//}
+      public VolumeAggregationStream(string future_contract, string contract_type, DateTime range_beg, DateTime range_end, ViewStreamDatabase db) {
+         future_contract = future_contract.EndsWith("*") ? future_contract : future_contract + "*";
+         ViewStream r;
+         r = new FutureHistoricalBarsStream(future_contract,
+                                             range_beg.ToString(), range_end.ToString());
+         r = new ContractExpanderStream(r, "symbol");
+         r = new FilterStream(r, $"contract_type='{contract_type}'");
+         r = new NonIncGroupByStream(r, "symbol", "volume:Sum", "ts_event");
+         r = new NonIncGroupByStream(r, "volume_sum:Max");
+         r = new NewProjectStream(r, "volume_sum_max:volume symbol !");
+         mySetRealSource(r);
+      }
+   }
+}

src/DataStreamSuite/SharedStreams/Business/MarketData/BookStream.cs

@@ -21,7 +21,7 @@ namespace BCG {
       private DateTime _last_poll_time = DateTime.MinValue;
       private ViewStreamDatabase _vsd;
       private int _id = 0;
-      IReadOnlyDictionary<SecurityKey, BookTuple>? _cached_state = null;
+      IReadOnlyDictionary<SecurityKey, BookTuple> _cached_state = null;
       MarketDataClient _md_client;
       readonly Stopwatch _timer = new();
       private readonly IBlueTableSchema _bbo_schema = BlueDataUtility.form_schema("books",
@@ -73,8 +73,14 @@ namespace BCG {
 
       private Dictionary<SecurityKey, Dictionary<int, int>> _id_map = new Dictionary<SecurityKey, Dictionary<int, int>>();
       private int _get_id(SecurityKey skey, int depth) {
-         var d_map = _id_map.GetOrAdd(skey, _ => new() { {depth, _id++} });
-         if (!d_map.TryGetValue(depth, out int rv)) {
+         Dictionary<int, int> d_map = null;
+         if (!_id_map.TryGetValue(skey, out d_map)) {
+            d_map = new Dictionary<int, int>();
+            d_map.Add(depth, _id++);
+            _id_map.Add(skey, d_map);
+         }
+         int rv = 0;
+         if (!d_map.TryGetValue(depth, out rv)) {
             rv = _id++;
             d_map.Add(depth, rv);
          }

src/DataStreamSuite/SharedStreams/Business/MarketData/Charting/VWAPStream.cs

@@ -0,0 +1,117 @@
+// namespace bts.SilverPine {
+//    using bts;
+//    using BCG;
+//    using MongoDB.Driver.Linq;
+//    using System.ComponentModel.DataAnnotations;
+//    using ZstdSharp.Unsafe;
+
+//    [DataStreamUserRespresentation("VWAP")]
+//    public class VWAPStream : ViewStream {
+//       // returns the VWAP stream for each contract 
+//       private static ViewStream build_vwap_stream(string symbol, DateTime begin_dt, DateTime end_dt) {
+//          var ohlcv_params = OHLCVStream.get_params(begin_dt, end_dt);
+//          ViewStream ohlc_str = new OHLCVStream(symbol, ohlcv_params);
+//          // stream is grouped per contract and then vwap is calculated per contract
+//          GlobalMapStream rs = new GlobalMapStream(ohlc_str, "ts_event", "contract",
+//          (rows, idx) => {
+//             Log.Gen.info($"VWAPStream.GlobalMapStream delegate n_rows: {rows.Count} idx: {idx}");
+//             IBlueRow ir = rows.ElementAt((int)idx);
+//             var sel_rows = rows.Skip((int)idx).Where(r => r.get_datetime("ts_event") >= end_dt && r.get_datetime("ts_event") <= begin_dt);
+//             var volumeprices = sel_rows.Select(r => Tuple.Create(r.get<int>("volume"), r.get<double>("close"))).ToList();
+//             double vwap = IndicatorCalc.calculate_VWAP(volumeprices);
+//             IBlueTableSchema schema = ir.schema.add_columns(Tuple.Create(typeof(double), "vwap"));
+//             MutableBlueRow nr = new MutableBlueRow(schema, ir);
+//             //vwap = -313.987;
+//             nr.setValue("vwap", vwap);
+//             return nr;
+//          });
+//          return rs;
+//       }
+//       // the args want dt1 > dt2, if its nto we, swap them
+//       static void swap_if_necessary(ref DateTime dt1, ref DateTime dt2) {
+//          if (dt1 < dt2) {
+//             DateTime aux = dt1;
+//             dt1 = dt2;
+//             dt2 = aux;
+//          }
+//       }
+
+//       // this assumes the ohlc_str has diff contracts
+//       public VWAPStream(string symbol_or_contract, DateTime begin_dt, DateTime end_dt) {
+//          swap_if_necessary(ref begin_dt, ref end_dt);
+//          Tuple<ContractUtils.ContractCategory, string> ccat_sym_contr = ContractUtils.get_symbol_category(symbol_or_contract);
+//          string symbol = String.Empty;
+//          symbol = ccat_sym_contr.Item2;
+//          var ohlcv_params = OHLCVStream.get_params(begin_dt, end_dt);
+//          ViewStream vwap_str = build_vwap_stream(symbol, begin_dt, end_dt);
+//          // vwap_str now has vwap for all contracts. if the arg was a symbol, we need to filter it for the active contract,
+//          // whihc we do with FutureActivesContractStream. If the arg was a contract we'll simply filter it
+//          ViewStream rs = vwap_str;
+//          switch (ccat_sym_contr.Item1) {
+//             case ContractUtils.ContractCategory.FutureString:
+//             case ContractUtils.ContractCategory.SpreadString:
+//                rs = new FilterStream(vwap_str, r => r.get<string>("contract").CompareTo(ccat_sym_contr.Item2) == 0);
+//                break;
+//             case ContractUtils.ContractCategory.Symbol:
+//                rs = new FuturesActiveContractStream(vwap_str);
+//                break;
+//          }
+//          rs = new FilterStream(rs, r => r.get_datetime("ts_event").CompareTo(begin_dt) == 0);
+//          rs = new NewProjectStream(rs, "! vwap");
+//          mySetRealSource(rs);
+//       }
+
+//       public VWAPStream(string symbol_or_contract, string begin_dt, string end_dt)
+//       :
+//       this(symbol_or_contract, DateTime.Parse(begin_dt), DateTime.Parse(end_dt)) { }
+//    }
+// }
+
+using System;
+
+using BCG;
+
+using bts.api;
+
+namespace bts.SilverPine {
+   /*
+      ----------------------------------------------------------------------
+                                  *Expression*
+      ----------------------------------------------------------------------
+          //nl("Find Settlement price of the SP500 future in the first week of 2023.")
+
+          r=FutureHistoricalBars("ES", "One_Day", "1/1/2024", "2/1/2024")
+          r=math(r, "vprice = volume * close")
+          r=aggregate(r, "vprice:Sum, volume:Sum")
+          r=math(r, "vwap = vprice_sum / volume_sum")
+          r=newProject(r, "vwap !")
+
+      ----------------------------------------------------------------------
+                                    *Schema*
+      ----------------------------------------------------------------------
+          id                              System.Int32
+          vwap                            System.Double
+   */
+
+   [DataStreamUserRespresentation("VWAP")]
+   public class VWAPStream : ViewStream {
+      public VWAPStream(string symbol_or_contract, DateTime beg_dt, DateTime end_dt)
+      :
+      this(symbol_or_contract, beg_dt, end_dt, ViewStreamDatabase.instance) { }
+
+      public VWAPStream(string symbol_or_contract, DateTime beg_dt, DateTime end_dt, ViewStreamDatabase db) {
+         var fc = new ContractUtils.FutureContract(symbol_or_contract);
+         ViewStream r = new FutureHistoricalBarsStream(fc, BarSize.ONE_DAY, new DateTimeRange(beg_dt, end_dt));
+
+         r = new MathStream(r,
+                 "vprice = volume * close");
+         r = new NonIncGroupByStream(r,
+                                      "vprice:Sum, volume:Sum");
+         r = new MathStream(r,
+                             "vwap = vprice_sum / volume_sum");
+         r = new NewProjectStream(r,
+                                   "vwap !");
+         mySetRealSource(r);
+      }
+   }
+}

src/DataStreamSuite/SharedStreams/Business/MarketData/FuturesActiveContractStream.cs

@@ -76,3 +76,35 @@ namespace bts.SilverPine {
       }
    }
 }
+
+
+namespace bts.SilverPine {
+   using BCG;
+
+   using bts.api;
+
+   using Newtonsoft.Json.Serialization;
+
+   [DataStreamUserRespresentation("FuturesContinousContract")]
+   public class FuturesContinousContractStream : ViewStream {
+      // the argument is symbol, like "ES" etc and not the contract itself
+      public FuturesContinousContractStream(string symbol) : this(symbol, ViewStreamDatabase.instance) { }
+      public FuturesContinousContractStream(string symbol, ViewStreamDatabase db) {
+         ViewStream r = new FuturesHistorical1DayBarsStream(symbol);
+         // filtering for futures only
+         r = new FilterStream(r, r => {
+            string contract = r.get<string>("symbol");
+            var fc = new ContractUtils.FutureContract(contract);
+            return fc.future_tradable_contract != null;
+         });
+         r = new NonIncGroupByStream(r,
+                                      "ts_event",
+                                      "volume:Max");
+         r = new NewProjectStream(r,
+                                   "ts_event symbol !");
+         r = new SortStream(r,
+                             "-ts_event");
+         mySetRealSource(r);
+      }
+   }
+}

src/DataStreamSuite/SharedStreams/Business/MarketData/FuturesRollStream.cs

@@ -1,18 +1,17 @@
+namespace bts.SilverPine {
+   using bts;
+   using BCG;
 
-//TODO: Remove
-//namespace bts.SilverPine {
-//   using bts;
-//   using BCG;
-
-//   // it uses the volume (better to use open interest perhap) to detect the rollover, ie when a new 
-//   // future exp becomes the highest volume one
-//   [DataStreamUserRespresentation("FuturesRoll")]
-//   public class FuturesRollStream : ViewStream {
-//      public FuturesRollStream(string future_symbol) {
-//         ViewStream rs = new FuturesContinousContractStream(future_symbol);
-//         rs = new ColumnEvolutionStream(rs, "ts_event", "symbol");
-//         rs = new NewProjectStream(rs, "symbol ts_event:roll_date !");
-//         mySetRealSource(rs);
-//      }
-//   }
-//}
+   // it uses the volume - better to use open interst perhap - to detect the rollover, is when a new 
+   // futuere exp becomes the highest volume one
+   [DataStreamUserRespresentation("FuturesRoll")]
+   public class FuturesRollStream : ViewStream {
+      public FuturesRollStream(string future_symbol) {
+         //ViewStream ohlcv_str = new OHLCVStream(future_symbol);
+         ViewStream rs = new FuturesActiveContractStream(future_symbol);
+         rs = new ColumnEvolutionStream(rs, "ts_event", "contract");
+         rs = new NewProjectStream(rs, "ts_event:roll_date !");
+         mySetRealSource(rs);
+      }
+   }
+}

src/DataStreamSuite/SharedStreams/Business/MarketData/InstrumentsStream.cs

@@ -1,9 +1,8 @@
-namespace bts.SilverPine {
+namespace BCG {
    using BCG.Utilities.General;
 
    using bts.api;
    using bts.md;
-   using BCG;
 
    using Spark.Core.Reactive;
 
@@ -32,7 +31,6 @@ namespace bts.SilverPine {
                                                 Tuple.Create("sec_type", TypeCode.String),
                                                 Tuple.Create("venue", TypeCode.String),
                                                 Tuple.Create("symbol", TypeCode.String),
-                                                Tuple.Create("currency", TypeCode.String),
                                                 Tuple.Create("expiration", TypeCode.DateTime),
                                                 Tuple.Create("strike", TypeCode.Double),
                                                 Tuple.Create("callput", TypeCode.String)
@@ -83,7 +81,7 @@ namespace bts.SilverPine {
          foreach (var kv in _cached_state) {
             SecurityKey skey = kv.Key;
             int id = Convert.ToInt32(skey);
-            SecurityImmutable sec = kv.Value;            
+            SecurityImmutable sec = kv.Value;
             bts.api.SecurityType sec_type = sec.security_type();
             object exp = DBNull.Value;
             if (sec_type == bts.api.SecurityType.Future || sec_type == bts.api.SecurityType.Option)
@@ -94,7 +92,7 @@ namespace bts.SilverPine {
             object cp = DBNull.Value;
             if (sec_type == bts.api.SecurityType.Option)
                cp = sec.Option.CallPut.ToString();
-            IBlueRow r = new BlueRow(_instrument_schema, id, skey, sec.security_desc(), sec.security_type().ToString(), sec.display_exchange().ToString(), sec.Symbol, sec.Currency, exp, strike, cp);
+            IBlueRow r = new BlueRow(_instrument_schema, id, skey, sec.security_desc(), sec.security_type().ToString(), sec.display_exchange().ToString(), sec.Symbol, exp, strike, cp);
             new_state.addRow(r);
          }
          DataPacket dp = null;
@@ -119,4 +117,4 @@ namespace bts.SilverPine {
          }
       }
    }
-}
+}

src/DataStreamSuite/SharedStreams/Business/MarketData/OHLCVStream.cs

@@ -174,10 +174,10 @@ namespace bts.SilverPine {
                // TODO: assuming 1 day only for historic               
                string table_name = get_table_from_barsize(ohlcv_params.bar_size, symbol);// $"{symbol}_OHLCV_1DAY";
                                                                                          //string table_name = $"{symbol}_{ohlcv_params.bar_size}";
-               string db_name = _db_name;// $"Data Source={_location}{_db_name};Version=3";
+               string db_str = $"Data Source={_location}{_db_name};Version=3";
                string sql_str = $"select * from {table_name}";
                // "Data Source=C:\\Users\\m_tah\\development\\data\\stream_data.db;Version=3;"
-               ViewStream rs = new SQLiteStream(db_name, sql_str);
+               ViewStream rs = new SQLiteStream(db_str, sql_str);
                rs = new SchemaGuesserStream(rs);
                if (ohlcv_params.bar_size == OHLCVBarSize.ONE_DAY)
                   rs = new DateStandardizationStream(rs, "ts_event", DateConversion.ChopTime);
@@ -207,4 +207,4 @@ namespace bts.SilverPine {
 
 
    }
-}
+}

src/DataStreamSuite/SharedStreams/Business/MarketData/SettlementPriceStream.cs

@@ -1,27 +1,26 @@
-// TODO: Remove
-//namespace bts.SilverPine {
-//   using BCG;
+namespace bts.SilverPine {
+   using BCG;
 
-//   using bts;
-//   // used directly in blueorb
-//   [DataStreamUserRespresentation("Settlement")]
-//   public class SettlementPriceStream : ViewStream {
+   using bts;
 
-//      // public SettlementPriceStream(string future_name, string date_txt)
-//      // : this(future_name, date_txt, date_txt) { }
+   [DataStreamUserRespresentation("Settlement")]
+   public class SettlementPriceStream : ViewStream {
 
-//      // public SettlementPriceStream(string future_name, string begin_dt, string end_dt)
-//      // : this(future_name, DateTime.Parse(begin_dt).Date, DateTime.Parse(end_dt).Date) { }
+      public SettlementPriceStream(string future_name, string date_txt)
+      : this(future_name, date_txt, date_txt) { }
 
-//      public SettlementPriceStream(string future_contract, DateTime begin_dt)
-//      : this(future_contract, begin_dt, begin_dt) { }
+      public SettlementPriceStream(string future_name, string begin_dt, string end_dt)
+      : this(future_name, DateTime.Parse(begin_dt).Date, DateTime.Parse(end_dt).Date) { }
 
-//      public SettlementPriceStream(string future_contract, DateTime begin_dt, DateTime end_dt) {
-//         ContractUtils.FutureContract fc = new ContractUtils.FutureContract(future_contract);
-//         ViewStream rs = new FutureHistoricalBarsStream(fc, BarSize.ONE_DAY, new DateTimeRange(begin_dt, end_dt));
-//         rs = new NewProjectStream(rs, "symbol ts_event:date close:settlement !");
-//         mySetRealSource(rs);
-//      }
+      public SettlementPriceStream(string symbol_or_contract, DateTime begin_dt)
+      : this(symbol_or_contract, begin_dt, begin_dt) { }
 
-//   }
-//}
+      public SettlementPriceStream(string symbol_or_contract, DateTime begin_dt, DateTime end_dt) {
+         ContractUtils.FutureContract fc = new ContractUtils.FutureContract(symbol_or_contract);
+         ViewStream rs = new FutureHistoricalBarsStream(fc, BarSize.ONE_DAY, new DateTimeRange(begin_dt, end_dt));
+         rs = new NewProjectStream(rs, "symbol ts_event:date close:settlement !");
+         mySetRealSource(rs);
+      }
+
+   }
+}

src/DataStreamSuite/SharedStreams/Business/MarketData/SyntheticSpreadStream.cs

@@ -1,125 +0,0 @@
-using System;
-using BCG;
-
-namespace bts.SilverPine
-{
-/*
-   ----------------------------------------------------------------------
-                               *Expression*
-   ----------------------------------------------------------------------
-       s1 = filter(bbo(), "sec = 'ETH-USDT-SWAP'")
-       s1 = addColumns(s1, "ljc:0")
-       s2 = filter(bbo(), "sec = 'ETH-USDT-SWAP'")
-       s2 = addColumns(s2, "rjc:0")
-       
-       s = fastJoin(s1, "ljc", "sec:s1 bid_price:s1_bid_price bid_sz:s1_bid_sz ask_price:s1_ask_price ask_sz:s1_ask_sz", 
-                    s2, "rjc", "sec:s2 bid_price:s2_bid_price bid_sz:s2_bid_sz ask_price:s2_ask_price ask_sz:s2_ask_sz")
-       
-
-   ----------------------------------------------------------------------
-                                 *Schema*
-   ----------------------------------------------------------------------
-       id                              System.Int32
-       s1                              System.String
-       s1_bid_price                    System.Double
-       s1_bid_sz                       System.Single
-       s1_ask_price                    System.Double
-       s1_ask_sz                       System.Single
-       s2                              System.String
-       s2_bid_price                    System.Double
-       s2_bid_sz                       System.Single
-       s2_ask_price                    System.Double
-       s2_ask_sz                       System.Single
-*/
-
-
-   // Combines the market data of spread legs into one stream, lined up nicely in a row.
-   // this is useful for operations like finding weights, that need leg market data organized together succintly.
-   public class SyntheticSpreadLegsStream : ViewStream 
-   {
-      public SyntheticSpreadLegsStream(string s1, string s2) : 
-         this(s1, s2, new BBOStream(ViewStreamDatabase.instance), ViewStreamDatabase.instance) {}
-
-      public SyntheticSpreadLegsStream(string s1, string s2, ViewStream bbo_stream, ViewStreamDatabase vsb)
-      {
-         ViewStream vs1;
-         ViewStream vs2;
-         ViewStream rs;
-
-         vs1 = new FilterStream (new BBOStream (),
-                                $"sec = '{s1}'");
-         vs1 = new AddColumnsStream (vs1,
-                                    "ljc:0");
-         vs2 = new FilterStream (bbo_stream,
-                                $"sec = '{s2}'");
-         vs2 = new AddColumnsStream (vs2,
-                                    "rjc:0");
-         rs = new NonIncJoinStream (vs1,
-                                   "ljc",
-                                   "sec:s1 bid_price:s1_bid_price bid_sz:s1_bid_sz ask_price:s1_ask_price ask_sz:s1_ask_sz",
-                                   vs2,
-                                   "rjc",
-                                   "sec:s2 bid_price:s2_bid_price bid_sz:s2_bid_sz ask_price:s2_ask_price ask_sz:s2_ask_sz");
-         mySetRealSource(rs);
-      }
-   }
-   public class SyntheticSpreadStream : ViewStream{
-      public SyntheticSpreadStream (string s1, string s2, double w1, double w2)
-      : this(s1, s2, w1, w2, new BBOStream(ViewStreamDatabase.instance), ViewStreamDatabase.instance){}
-      public SyntheticSpreadStream (string s1, string s2, double w1, double w2, ViewStream bbo_stream, ViewStreamDatabase vsd){
-         ViewStream rs = new SyntheticSpreadLegsStream(s1, s2, bbo_stream, vsd);
-
-         Func<IBlueTableSchema, IBlueTableSchema> get_schema = (inp_schema) => {
-            IBlueTableSchema rv = inp_schema;
-            rv = rv.add_column(TypeCode.Double, "spread_bid_price");
-            rv = rv.add_column(TypeCode.Double, "spread_bid_sz");
-            rv = rv.add_column(TypeCode.Double, "spread_ask_price");
-            rv = rv.add_column(TypeCode.Double, "spread_ask_sz");
-            return rv;
-         };
-
-         rs = new MapStream(rs, (inp_schema) => {
-           return get_schema(inp_schema);
-         }, 
-         (r) => {
-              double s1_bid_price = r.get<double>("s1_bid_price");
-              double s1_bid_sz = r.get<double>("s1_bid_sz");
-              double s1_ask_price = r.get<double>("s1_ask_price");
-              double s1_ask_sz = r.get<double>("s1_ask_sz");
-               
-              double s2_bid_price = r.get<double>("s2_bid_price");
-              double s2_bid_sz = r.get<double>("s2_bid_sz");
-              double s2_ask_price = r.get<double>("s2_ask_price");
-              double s2_ask_sz = r.get<double>("s2_ask_sz");
-               
-              (double spread_bid_price, double spread_ask_price, double spread_bid_sz, double spread_ask_sz) =   
-              ComputeSpreadBidAsk(s1_bid_price, s1_ask_price, s2_bid_price, s2_ask_price, s1_bid_sz, s1_ask_sz, s2_bid_sz, s2_ask_sz, w1, w2);
-              IBlueTableSchema out_schema = get_schema(r.schema);
-              MutableBlueRow rv = new MutableBlueRow(out_schema, r);
-              rv.setValue("spread_bid_price", spread_bid_price);
-              rv.setValue("spread_ask_price", spread_ask_price);
-              rv.setValue("spread_bid_sz", spread_bid_sz);
-              rv.setValue("spread_ask_sz", spread_ask_sz);
-              return rv;
-              }
-           );
-         mySetRealSource(rs);
-
-      }
-      public static (double SpreadBid, double SpreadAsk, double SpreadBidQty, double SpreadAskQty) ComputeSpreadBidAsk(
-        double s1Bid, double s1Ask,
-        double s2Bid, double s2Ask,
-        double s1BidQty, double s1AskQty,
-        double s2BidQty, double s2AskQty,
-        double w1, double w2)
-    {
-        double spreadBid = (w1 * s1Bid) - (w2 * s2Ask);
-        double spreadAsk = (w1 * s1Ask) - (w2 * s2Bid);
-
-        double spreadBidQty = Math.Min(s1BidQty / w1, s2AskQty / w2);
-        double spreadAskQty = Math.Min(s1AskQty / w1, s2BidQty / w2);
-
-        return (spreadBid, spreadAsk, spreadBidQty, spreadAskQty);
-    }
-   }
-}

src/DataStreamSuite/SharedStreams/NL/DSL_master_learnign_prompt.md

@@ -1,571 +0,0 @@
-# BlueOrbDSL Master Learning Prompt v1.1 (Optimized)
-
-**This version is the *final, frozen* canonical Master Learning Prompt for BlueOrbDSL model training and generation.**  
-**No further edits are required. This is the single source of truth for Learning Mode.**  
-**Reviewed and validated as optimal on \[insert date here, e.g. June 9, 2025\] after full Meta Mode review.**
-
-**Change Log — v1 → v1.1**  
-(June 7, 2025\)
-
-✅ Applied full Meta Mode review recommendations:
-
-- Standardized **Arguments** section (bullet lists, first arg \= `parent_stream` for operators)  
-- Added **Adds Columns?** and **Modifies Schema?** rows for *all* streams  
-- Clarified **Special Notes** for all streams  
-- Clearly stated UnionStream schema requirements  
-- Explicitly documented JoinStream as inner join, with left/right prefix behavior  
-- Clarified behavior of **TimeBasedRollingFilterStream**  
-- Changed DateTimeExpanderStream new columns to a bullet list for clarity  
-- Standardized all **Example Usage** to fenced `dsl` code blocks  
-- Shortened **PythonDataframeStream** example; big one should go to appendix if needed  
-- Minor language and consistency cleanups across stream definitions
-
-**This version is now ready to serve as the authoritative MLP for BlueOrbDSL training and generation.**  
-When instructed to "Switch to Learning Mode," this will become the model’s single source of truth.
-
----
-
-# Table of Contents
-
-1. Preamble  
-2. Streams Overview  
-3. Data Sources  
-   3.1 FutureHistoricalBarsStream  
-   3.2 LiveBBOStream  
-   3.3 LiteralStream  
-4. Operators  
-   4.1 FilterStream  
-   4.2 ProjectStream  
-   4.3 GroupByStream  
-   4.4 SingleRowMathStream  
-   4.5 MultiRowRollingMathStream  
-   4.6 UnionStream  
-   4.7 JoinStream  
-   4.8 DateTimeExpanderStream  
-   4.9 ContractExpanderStream  
-   4.10 CountStream  
-   4.11 TimeSeriesStream  
-   4.12 TimeBasedRollingFilterStream  
-   4.13 PythonDataframeStream  
-5. Reserved Words & Enumerations  
-6. Common Pitfalls
-
----
-
-## 1 Preamble
-
-BlueOrbDSL turns natural-language finance questions into **stream pipelines** that operate on tabular market data (historical or real-time).  
-Each *stream* is either a **Data Source** (fetches data) or an **Operator** (transforms data).  
-Streams compose: the output table of one becomes the input of the next.
-
-**Convention:** For all operators, the **first argument** is always the `parent_stream`. Subsequent arguments are constructor parameters.
-
----
-
-## 2 Streams Overview
-
-All stream definitions follow this table layout:
-
-| Field                | Meaning                                                |
-| :------------------- | :----------------------------------------------------- |
-| **Type**             | DataSource / Operator                                  |
-| **Purpose**          | One-sentence description                               |
-| **Arguments**        | Bullet list; first arg \= parent\_stream (if Operator) |
-| **Output Schema**    | Columns created or removed                             |
-| **Adds Columns?**    | Yes / No                                               |
-| **Modifies Schema?** | Yes / No                                               |
-| **Example Usage**    | Fenced `dsl` block                                     |
-| **Special Notes**    | Constraints / caveats                                  |
-
----
-
-## 3 Data Sources
-
-### 3.1 FutureHistoricalBarsStream
-
-| Field                | Description                                                                                                                                                                                                |
-| :------------------- | :--------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------- |
-| **Type**             | DataSource                                                                                                                                                                                                 |
-| **Purpose**          | Fetch historical OHLCV bars for futures contracts                                                                                                                                                          |
-| **Arguments**        | • `contract` — continuous (`ES`), specific (`ESZ9`), or wildcard (`ES*`) • \``bar_size`\` — ONE\_DAY, ONE\_MIN • `` `range_begin` `` — ISO datetime, optional • `` `range_end` `` — ISO datetime, optional |
-| **Output Schema**    | `id, ts_event, date, contract, open, high, low, close, volume`                                                                                                                                             |
-| **Adds Columns?**    | Yes                                                                                                                                                                                                        |
-| **Modifies Schema?** | No                                                                                                                                                                                                         |
-| **Example Usage**    | `bars = FutureHistoricalBarsStream("ES", "ONE_DAY", "2023-01-01", "2023-12-31")`                                                                                                                           |
-| **Special Notes**    | Whereas the ts\_event column is the timestamp of the column, date is the date of the day.                                                                                                                  |
-
-### 3.2 LiveBBOStream
-
-| Field                | Description                                                                                                                                                                |
-| :------------------- | :------------------------------------------------------------------------------------------------------------------------------------------------------------------------- |
-| **Type**             | DataSource                                                                                                                                                                 |
-| **Purpose**          | Stream live best-bid/offer quotes                                                                                                                                          |
-| **Arguments**        | • `contract` — e.g. `"NGU5"` or `"CL*"`                                                                                                                                    |
-| **Output Schema**    | `ts_event, contract, bid_price, bid_size, ask_price, ask_size`                                                                                                             |
-| **Adds Columns?**    | Yes                                                                                                                                                                        |
-| **Modifies Schema?** | No                                                                                                                                                                         |
-| **Example Usage**    | `bbo = LiveBBOStream("NGU5")`                                                                                                                                              |
-| **Special Notes**    | LiveBBOStream produces the latest value of the bid/ask price and size. It does not maintain history. Each new row overwrites the old one. For history use TimeSeriesStream |
-
-### 3.3 LiteralStream
-
-| Field                | Description                                                                                                       |
-| :------------------- | :---------------------------------------------------------------------------------------------------------------- |
-| **Type**             | DataSource                                                                                                        |
-| **Purpose**          | Inject small static tables via JSON records                                                                       |
-| **Arguments**        | • `json_text` — string                                                                                            |
-| **Output Schema**    | Inferred from JSON keys                                                                                           |
-| **Adds Columns?**    | Yes                                                                                                               |
-| **Modifies Schema?** | No                                                                                                                |
-| **Example Usage**    | `meta = LiteralStream('[{"event":"OPEC","date":"2024-06-01"}]')`                                                  |
-| **Special Notes**    | If you download the data from the internet, and want to use it conjunction with the BlueOrbDSL this can be useful |
-
----
-
-## 4 Operators
-
-### 4.1 FilterStream
-
-| Field                | Description                                                                                                                  |
-| :------------------- | :--------------------------------------------------------------------------------------------------------------------------- |
-| **Type**             | Operator                                                                                                                     |
-| **Purpose**          | Keep rows where a boolean condition is true                                                                                  |
-| **Arguments**        | • `parent_stream` •`` `condition` `` — string                                                                                |
-| **Output Schema**    | Same as input                                                                                                                |
-| **Adds Columns?**    | No                                                                                                                           |
-| **Modifies Schema?** | No                                                                                                                           |
-| **Special Notes**    | Condition syntax: use ASCII single quotes (`contract = 'ESZ9'`). Supports `and`, `or`, parentheses, equality and inequality. |
-
-### 4.2 ProjectStream
-
-| Field                | Description                                                                         |
-| :------------------- | :---------------------------------------------------------------------------------- |
-| **Type**             | Operator                                                                            |
-| **Purpose**          | Rename or drop columns                                                              |
-| **Arguments**        | • `parent_stream` • `` `column_map ``\` — string                                    |
-| **Output Schema**    | Renamed subset                                                                      |
-| **Adds Columns?**    | No                                                                                  |
-| **Modifies Schema?** | Yes                                                                                 |
-| **Example Usage**    | `clean = ProjectStream(bars, "close:settlement, volume")`                           |
-| **Special Notes**    | **Cannot add new columns or constants.** Use LiteralStream \+ JoinStream if needed. |
-
----
-
-### 4.3 GroupByStream
-
-| Field                | Description                                                                                            |
-| :------------------- | :----------------------------------------------------------------------------------------------------- |
-| **Type**             | Operator                                                                                               |
-| **Purpose**          | Aggregates rows by key (or globally)                                                                   |
-| **Arguments**        | • `parent_stream` • `` `group_by` `` — string (blank for global) • `` `aggregation_string` `` — string |
-| **Output Schema**    | Adds `<column>_<aggregation>` columns                                                                  |
-| **Adds Columns?**    | Yes                                                                                                    |
-| **Modifies Schema?** | Yes                                                                                                    |
-| **Example Usage**    | `daily = GroupByStream(bars, "date", "volume:Sum")`                                                    |
-| **Special Notes**    | None                                                                                                   |
-
-### 4.4 SingleRowMathStream
-
-| Field                | Description                                                                  |
-| :------------------- | :--------------------------------------------------------------------------- |
-| **Type**             | Operator                                                                     |
-| **Purpose**          | Add new columns from row-wise math expressions                               |
-| **Arguments**        | • `parent_stream` • `` `math_expression` `` — string                         |
-| **Output Schema**    | Adds new column(s)                                                           |
-| **Adds Columns?**    | Yes                                                                          |
-| **Modifies Schema?** | No                                                                           |
-| **Example Usage**    | `r = SingleRowMathStream(bars, "daily_return = close - open")`               |
-| **Special Notes**    | Operates on one row at a time only. Cannot reference values from other rows. |
-
-### 4.5 MultiRowRollingMathStream
-
-| Field                | Description                                                                                                                                 |
-| :------------------- | :------------------------------------------------------------------------------------------------------------------------------------------ |
-| **Type**             | Operator                                                                                                                                    |
-| **Purpose**          | Aggregates over rolling window of rows                                                                                                      |
-| **Arguments**        | • `parent_stream` • `` `aggregation_string` `` — string • `` `sort_column` `` — string with \+/- prefix • `` `rolling_window_size` `` — int |
-| **Output Schema**    | Adds `<column>_<aggregation>` columns                                                                                                       |
-| **Adds Columns?**    | Yes                                                                                                                                         |
-| **Modifies Schema?** | Yes                                                                                                                                         |
-| **Example Usage**    | `ma20 = MultiRowRollingMathStream(bars, "close:Mean", "-ts_event", 20)`                                                                     |
-| **Special Notes**    | sort\_column should be prefixed with \+ (ascending) or \- (descending). Example: "-ts\_event".                                              |
-
-### 4.6 UnionStream
-
-| Field                | Description                                              |
-| :------------------- | :------------------------------------------------------- |
-| **Type**             | Operator                                                 |
-| **Purpose**          | Append rows from two streams                             |
-| **Arguments**        | • `parent_stream1` • `parent_stream2`                    |
-| **Output Schema**    | Same as inputs                                           |
-| **Adds Columns?**    | No                                                       |
-| **Modifies Schema?** | No                                                       |
-| **Example Usage**    | `unioned = UnionStream(bars1, bars2)`                    |
-| **Special Notes**    | **Schemas must match exactly (column names AND types).** |
-
-### 4.7 JoinStream
-
-| Field                | Description                                                                                                                                                           |
-| :------------------- | :-------------------------------------------------------------------------------------------------------------------------------------------------------------------- |
-| **Type**             | Operator                                                                                                                                                              |
-| **Purpose**          | Join two streams on equality of selected columns                                                                                                                      |
-| **Arguments**        | • `` `left_parent_stream` ``   • `` `left_join_cols` `` — comma-separated string   • `` `right_parent_stream` ``   • `` `right_join_cols` `` — comma-separated string |
-| **Output Schema**    | Combined columns from both streams                                                                                                                                    |
-| **Adds Columns?**    | Yes                                                                                                                                                                   |
-| **Modifies Schema?** | Yes                                                                                                                                                                   |
-| **Example Usage**    | `joined = JoinStream(a, "ts_event", b, "ts_event")`                                                                                                                   |
-| **Special Notes**    | Always performs an **inner join** on equality. If columns overlap, prefix applied (`left_`, `right_`). If join columns args left empty, does cartesian product        |
-
-### 4.8 DateTimeExpanderStream
-
-| Field                | Description                                                                                                                        |
-| :------------------- | :--------------------------------------------------------------------------------------------------------------------------------- |
-| **Type**             | Operator                                                                                                                           |
-| **Purpose**          | Adds columns with date/time components from a datetime column                                                                      |
-| **Arguments**        | • `parent_stream` • `datetime_column` — string                                                                                     |
-| **Output Schema**    | Adds: • \`\_date\` • \`\_day\_of\_week\` • \`\_day\_of\_month\` • \`\_month\` • \`\_hour\` • \`\_minute\` • \`\_seconds\_of\_day\` |
-| **Adds Columns?**    | Yes                                                                                                                                |
-| **Modifies Schema?** | Yes                                                                                                                                |
-| **Example Usage**    | `expanded = DateTimeExpanderStream(bars, "ts_event")`                                                                              |
-| **Special Notes**    | None                                                                                                                               |
-
-### 4.9 ContractExpanderStream
-
-| Field                | Description                                           |
-| :------------------- | :---------------------------------------------------- |
-| **Type**             | Operator                                              |
-| **Purpose**          | Adds metadata columns based on `contract`             |
-| **Arguments**        | • `parent_stream` • contract\_column — string         |
-| **Output Schema**    | Adds: `• contract_type`                               |
-| **Adds Columns?**    | Yes                                                   |
-| **Modifies Schema?** | Yes                                                   |
-| **Example Usage**    | `expanded = ContractExpanderStream(bars, "contract")` |
-| **Special Notes**    | None                                                  |
-
-### 4.10 CountStream
-
-| Field                | Description                       |
-| :------------------- | :-------------------------------- |
-| **Type**             | Operator                          |
-| **Purpose**          | Count number of rows              |
-| **Arguments**        | • `parent_stream`                 |
-| **Output Schema**    | `id, count`                       |
-| **Adds Columns?**    | Yes                               |
-| **Modifies Schema?** | Yes                               |
-| **Example Usage**    | `total_count = CountStream(bars)` |
-| **Special Notes**    | None                              |
-
-### 4.11 TimeSeriesStream
-
-| Field                | Description                                                          |
-| :------------------- | :------------------------------------------------------------------- |
-| **Type**             | Operator                                                             |
-| **Purpose**          | Append each update from parent stream into time series               |
-| **Arguments**        | • `parent_stream` • \``buffer_size`\` — int                          |
-| **Output Schema**    | Parent schema \+ `timestamp` (if not already present)                |
-| **Adds Columns?**    | Yes                                                                  |
-| **Modifies Schema?** | No                                                                   |
-| **Example Usage**    | `ts_bbo = TimeSeriesStream(bbo, 10000)`                              |
-| **Special Notes**    | Append-only stream, does not deduplicate or collapse repeated values |
-
-### 4.12 TimeBasedRollingFilterStream
-
-| Field                | Description                                                                  |
-| :------------------- | :--------------------------------------------------------------------------- |
-| **Type**             | Operator                                                                     |
-| **Purpose**          | Keep rows newer than rolling window of N seconds                             |
-| **Arguments**        | • `parent_stream` • \``no_of_seconds`\` — int                                |
-| **Output Schema**    | Same as input                                                                |
-| **Adds Columns?**    | No                                                                           |
-| **Modifies Schema?** | No                                                                           |
-| **Example Usage**    | `recent = TimeBasedRollingFilterStream(trades, 60)`                          |
-| **Special Notes**    | Drops rows older than `no_of_seconds` compared to current max timestamp seen |
-
-### 4.13 PythonDataframeStream
-
-| Field                | Description                                                                                                             |
-| :------------------- | :---------------------------------------------------------------------------------------------------------------------- |
-| **Type**             | Operator                                                                                                                |
-| **Purpose**          | Apply arbitrary Python transformation to DataFrame                                                                      |
-| **Arguments**        | • `parent_stream` • `` `python_function` `` — string                                                                    |
-| **Output Schema**    | Defined by function                                                                                                     |
-| **Adds Columns?**    | Yes/No — depends on function                                                                                            |
-| **Modifies Schema?** | Yes/No — depends on function                                                                                            |
-| **Example Usage**    | `r = PythonDataframeStream(bars, "def transform(df): return df[df.volume > 1e6]")`                                      |
-| **Special Notes**    | **Use only as last resort.** Function should be small and well-documented. Avoid large multi-step logic where possible. |
-
----
-
-## 5 Reserved Words & Enumerations
-
-- **Bar\_size:** `ONE_DAY`, `ONE_MIN`  
-- **Aggregation Operations:** `Mean`, `Median`, `Mode`, `Sum`, `Max`, `Min`, `StdDev`, `Range`, `First`, `Last`, `EdgeDiff`, `EdgeDiffPercentage`
-
----
-
-## 6 Common Pitfalls
-
-| Pitfall                                         | Correct Approach                                     |
-| :---------------------------------------------- | :--------------------------------------------------- |
-| Adding literal columns in ProjectStream         | Use LiteralStream → JoinStream                       |
-| Smart quotes in conditions                      | Use `'...'` ASCII quotes                             |
-| Large functions in PythonDataframeStream        | Keep minimal or move to appendix                     |
-| Ambiguous TimeBasedRollingFilterStream behavior | Now clearly stated: drops rows older than N seconds. |
-
-## 7 Reinforced Learning DSL Examples
-
-This section contains **complete worked examples** of BlueOrbDSL pipelines answering complex natural language queries.
-
-Each example shows:
-
-- The **NL Query** (natural language question)  
-- The **DSL Solution** in full  
-- Optional **Notes**
-
----
-
-### Example 1 — On which dates did SP 500 Futures open 2% higher than the previous day's close?
-
-#### DSL Solution
-
-```
-bars = FutureHistoricalBarsStream("ES", "ONE_DAY", "1900-01-01T00:00:00", "2100-12-31T23:59:59")
-rolling_bars = MultiRowRollingMathStream(bars, "close:First, open:Last", "+ts_event", 2)
-percent_change = SingleRowMathStream(rolling_bars, "percent_change = 100 * (open_last - close_first) / close_first")
-filtered_bars = FilterStream(percent_change, "percent_change >= 2")
-result = ProjectStream(filtered_bars, "ts_event, percent_change")
-```
-
----
-
-### Example 2 — What is the average price movement for e-minis on Wednesday between 9:30 am and 9:40 AM CST?
-
-#### DSL Solution
-
-```
-bars = FutureHistoricalBarsStream("ES", "ONE_MIN", "1900-01-01T00:00:00", "2100-12-31T23:59:59")
-expanded_bars = DateTimeExpanderStream(bars, "ts_event")
-filtered_bars = FilterStream(expanded_bars, "ts_event_day_of_week = 'Wednesday' and ts_event_hour = 9 and ts_event_minute >= 30 and ts_event_minute < 40")
-price_movement = SingleRowMathStream(filtered_bars, "movement = close - open")
-average_movement = GroupByStream(price_movement, "", "movement:Mean")
-result = ProjectStream(average_movement, "movement_mean:average_price_movement")
-```
-
----
-
-### Example 3 — What did the volume and price of ZN in the 5 minutes following the last 10 FOMC rates?
-
-#### DSL Solution
-
-```
-bars_1 = FutureHistoricalBarsStream("ZN", "ONE_MIN", "2023-11-01T14:00:00", "2023-11-01T14:05:00")
-bars_2 = FutureHistoricalBarsStream("ZN", "ONE_MIN", "2023-09-20T14:00:00", "2023-09-20T14:05:00")
-bars_3 = FutureHistoricalBarsStream("ZN", "ONE_MIN", "2023-07-26T14:00:00", "2023-07-26T14:05:00")
-bars_4 = FutureHistoricalBarsStream("ZN", "ONE_MIN", "2023-06-14T14:00:00", "2023-06-14T14:05:00")
-bars_5 = FutureHistoricalBarsStream("ZN", "ONE_MIN", "2023-05-03T14:00:00", "2023-05-03T14:05:00")
-bars_6 = FutureHistoricalBarsStream("ZN", "ONE_MIN", "2023-03-22T14:00:00", "2023-03-22T14:05:00")
-bars_7 = FutureHistoricalBarsStream("ZN", "ONE_MIN", "2023-02-01T14:00:00", "2023-02-01T14:05:00")
-bars_8 = FutureHistoricalBarsStream("ZN", "ONE_MIN", "2022-12-14T14:00:00", "2022-12-14T14:05:00")
-bars_9 = FutureHistoricalBarsStream("ZN", "ONE_MIN", "2022-11-02T14:00:00", "2022-11-02T14:05:00")
-bars_10 = FutureHistoricalBarsStream("ZN", "ONE_MIN", "2022-09-21T14:00:00", "2022-09-21T14:05:00")
-
-all_bars = UnionStream(bars_1, bars_2)
-all_bars = UnionStream(all_bars, bars_3)
-all_bars = UnionStream(all_bars, bars_4)
-all_bars = UnionStream(all_bars, bars_5)
-all_bars = UnionStream(all_bars, bars_6)
-all_bars = UnionStream(all_bars, bars_7)
-all_bars = UnionStream(all_bars, bars_8)
-all_bars = UnionStream(all_bars, bars_9)
-all_bars = UnionStream(all_bars, bars_10)
-
-grouped_data = GroupByStream(all_bars, "date", "open:First, close:Last, high:Max, low:Min, volume:Sum")
-result = ProjectStream(grouped_data, "ts_event_date, open_first:5_min_open, close_last:5_min_close, high_max:5_min_high, low_min:5_min_low, volume_sum:5_min_volume")
-```
-
----
-
-### Example 4 — How much volume of RBOB typically trades during the settlement window?
-
-#### DSL Solution
-
-```
-rbob_bars = FutureHistoricalBarsStream("RB", "ONE_MIN", "2023-01-01T00:00:00", "2023-12-31T23:59:59")
-expanded_bars = DateTimeExpanderStream(rbob_bars, "ts_event")
-settlement_window = FilterStream(expanded_bars, "(ts_event_hour = 14 and ts_event_minute >= 28) or (ts_event_hour = 14 and ts_event_minute < 30)")
-typical_settlement_volume = GroupByStream(settlement_window, "ts_event_date", "volume:Sum")
-result = ProjectStream(typical_settlement_volume, "ts_event_date, volume_sum:daily_settlement_volume")
-```
-
----
-
-### Example 5 — How much SP Futures during the settlement window, average per day, during the 2 week windows before future expiration, in 2023?
-
-#### DSL Solution
-
-```
-sp_bars = FutureHistoricalBarsStream("ES", "ONE_MIN", "2023-01-01T00:00:00", "2023-12-31T23:59:59")
-expanded_bars = DateTimeExpanderStream(sp_bars, "ts_event")
-settlement_window = FilterStream(expanded_bars, "(ts_event_hour = 15 and ts_event_minute >= 58) or (ts_event_hour = 16 and ts_event_minute < 1)")
-filtered_bars = FilterStream(settlement_window, "(ts_event_date >= '2023-03-03' and ts_event_date < '2023-03-17') or (ts_event_date >= '2023-06-02' and ts_event_date < '2023-06-16') or (ts_event_date >= '2023-09-01' and ts_event_date < '2023-09-15') or (ts_event_date >= '2023-12-01' and ts_event_date < '2023-12-15')")
-daily_volume = GroupByStream(filtered_bars, "ts_event_date", "volume:Sum")
-average_daily_volume = GroupByStream(daily_volume, "", "volume_sum:Mean")
-result = ProjectStream(average_daily_volume, "volume_sum_mean:average_daily_volume")
-```
-
----
-
-### Example 6 — Find the difference between the number of days in 2023 that SP 500 went up 2% and when it went down 2%
-
-#### DSL Solution
-
-```
-bars = FutureHistoricalBarsStream("ES", "ONE_DAY", "2023-01-01T00:00:00", "2023-12-31T23:59:59")
-percentage_change = SingleRowMathStream(bars, "percent_change = 100 * (close - open) / open")
-days_up_2_percent = FilterStream(percentage_change, "percent_change >= 2")
-days_up_count = CountStream(days_up_2_percent)
-days_down_2_percent = FilterStream(percentage_change, "percent_change <= -2")
-days_down_count = CountStream(days_down_2_percent)
-joined_counts = JoinStream(days_up_count, "", days_down_count, "")
-difference_result = SingleRowMathStream(joined_counts, "difference = days_up_count - days_down_count")
-result = ProjectStream(difference_result, "difference")
-```
-
----
-
-### Example 7 — Monitor which CL contracts (CL) are trading with bid size \>= 1000, and record the last 1000 events\*
-
-#### DSL Solution
-
-```
-bbo = LiveBBOStream("CL*")
-large_bid = FilterStream(bbo, "bid_size >= 1000")
-large_bid_ts = TimeSeriesStream(large_bid, 1000)
-result = ProjectStream(large_bid_ts, "timestamp, ts_event, contract, bid_size, bid_price, ask_price")
-```
-
----
-
-### Example 8 — Identify FutureSpreads in CrudeOil and monitor their spread width
-
-#### DSL Solution
-
-```
-bbo = LiveBBOStream("CL*")
-expanded_bbo = ContractExpanderStream(bbo, "contract")
-future_spreads = FilterStream(expanded_bbo, "contract_type = 'FutureSpread'")
-spread_stream = SingleRowMathStream(future_spreads, "spread = ask_price - bid_price")
-result = ProjectStream(spread_stream, "ts_event, contract, spread")
-```
-
----
-
-### Example 9 — Track when bid prices in NaturalGas Futures cross above 3.00 and keep only the last 10 seconds of such events
-
-#### DSL Solution
-
-```
-bbo = LiveBBOStream("NG")
-high_bid = FilterStream(bbo, "bid_price > 3.00")
-high_bid_ts = TimeSeriesStream(high_bid, 10000)
-recent_high_bid = TimeBasedRollingFilterStream(high_bid_ts, 10)
-result = ProjectStream(recent_high_bid, "ts_event, contract, bid_price, bid_size, ask_price")
-```
-
----
-
-### Example 11 — Detect sequences in NaturalGas future where bid\_price increased 3 times consecutively within a rolling 30-second window
-
-#### DSL Solution
-
-```
-bbo = LiveBBOStream("NG")
-bbo_ts = TimeSeriesStream(bbo, 10000)
-recent_bbo = TimeBasedRollingFilterStream(bbo, 30)  // 30-second window of updates
-
-result = PythonDataframeStream(recent_bbo, "
-def transform(df):
-    import pandas as pd
-    df = df.sort_values('ts_event')
-    df['bid_increase'] = df['bid_price'].diff() > 0
-    df['consec_increase'] = df['bid_increase'].rolling(window=3).sum()
-    # Find rows where we just reached 3 consecutive increases
-    df['trigger'] = (df['consec_increase'] == 3) & (df['consec_increase'].shift(1) < 3)
-    return df[df['trigger']][['ts_event', 'contract', 'bid_price', 'bid_size']]
-")
-```
-
----
-
-### Example 12 — When we get a 5+ level bid sweep in SP 500, show me how the bid behaves in the 10 minutes after — mean, stddev, min/max bid\_price
-
-#### DSL Solution
-
-```
-bbo = LiveBBOStream("ES")
-bbo_ts = TimeSeriesStream(bbo, 10000)
-recent_bbo = TimeSeriesStream(bbo, 5000)  // keep large enough window for event tracking
-
-result = PythonDataframeStream(recent_bbo, "
-def transform(df):
-    import pandas as pd
-    df = df.sort_values('ts_event')
-    rows = []
-    in_measurement = False
-    measurement_start_time = None
-    measurement_end_time = None
-    bid_prices_in_window = []
-    min_bid_size_threshold = 100  # defines what we consider 'stable large bid'
-    min_levels_stable = 5
-    prior_bid_sizes = []
-
-    for i, row in df.iterrows():
-        bid_price = row['bid_price']
-        bid_size = row['bid_size']
-        ts = pd.to_datetime(row['ts_event'])
-
-        # Maintain rolling window of prior bid_sizes
-        prior_bid_sizes.append(bid_size)
-        if len(prior_bid_sizes) > min_levels_stable:
-            prior_bid_sizes.pop(0)
-
-        if not in_measurement:
-            # Detect sweep trigger
-            if all(x > min_bid_size_threshold for x in prior_bid_sizes[:-1]) and bid_size < min_bid_size_threshold:
-                # Sweep detected → start measurement window
-                measurement_start_time = ts
-                measurement_end_time = measurement_start_time + pd.Timedelta(minutes=10)
-                bid_prices_in_window = []
-                in_measurement = True
-        else:
-            # Inside measurement window
-            if ts <= measurement_end_time:
-                bid_prices_in_window.append(bid_price)
-            else:
-                # Window complete → emit stats
-                if bid_prices_in_window:
-                    mean_bid = pd.Series(bid_prices_in_window).mean()
-                    stddev_bid = pd.Series(bid_prices_in_window).std()
-                    min_bid = pd.Series(bid_prices_in_window).min()
-                    max_bid = pd.Series(bid_prices_in_window).max()
-                    rows.append({
-                        'sweep_time': measurement_start_time,
-                        'measurement_end': measurement_end_time,
-                        'mean_bid_price': mean_bid,
-                        'stddev_bid_price': stddev_bid,
-                        'min_bid_price': min_bid,
-                        'max_bid_price': max_bid
-                    })
-                # Reset state
-                in_measurement = False
-                measurement_start_time = None
-                measurement_end_time = None
-                bid_prices_in_window = []
-                prior_bid_sizes = []
-
-    return pd.DataFrame(rows)
-")
-
-```
-
-*End of Section 7 — Reinforced Learning DSL Examples*  

src/DataStreamSuite/SharedStreams/NL/DSL_meta_prompt,md

@@ -1,27 +0,0 @@
-**Enter Meta Mode**  
- You are not learning the DSL yet.  
- You are reviewing the following **BlueOrbDSL Master Learning Prompt** (MLP) as a *training document for yourself*.
-
-Your goal is to optimize it to make it easier for you — or any future LLM — to learn from it and generate DSL code with high accuracy.
-
-Please analyze the MLP and provide:
-
-1. **Clarity review** — is each stream definition clear, unambiguous, easy to parse?
-
-2. **Consistency review** — do all stream definitions follow the same structure and style?
-
-3. **Terminology check** — are key terms (parent stream, arguments, schema impact, example usage, etc.) used consistently?
-
-4. **Structural feedback** — is the overall document structured in a way that supports learning (headings, bold, tables, code blocks)?
-
-5. **Formatting feedback** — are important warnings / "cannot do X" constraints highlighted clearly?
-
-6. **Missing info** — is anything missing from any stream definition that would help you understand how to use it?
-
-7. **Generation risk spots** — any parts where you foresee yourself making common mistakes due to ambiguity?
-
-**Do not attempt to "learn" this DSL yet.** Stay in Meta Mode and provide feedback as an *editor and optimizer*.
-
-When I later say:  
- → **Switch to Learning Mode — this is now the final BlueOrbDSL MLP** → you will forget your Meta comments and treat the MLP as the single authoritative source for DSL generation.
-

src/DataStreamSuite/SharedStreams/NL/FewShotPrompts.json

@@ -2,13 +2,9 @@
    "prompts": [
       {
          "category": "Settlement",
-         "description": "This is a few shot learning example for an NL to DSL. The DSL function in question is Settlement (symbol_or_contract, datetime1). The first argument is a future (like CL, for crude oil, or ES for SP 500) or a specific future contract like crude oil Dec 24 future (CLZ4) or ESZ9 etc. The second argument is the date at which the settlement is to be fetched. There is an overloaded function, Settlement (symbol_or_contract, begin_dt, end_dt) which gets the settlements for a range pf dates from begin_dt to end_dt. You will need to convert statements  expressed as Today, Yesterday, Last Friday, First Friday of the month, Last Thursday of August, First day of last quarter etc to actual dates. The function Settlement's second and third arguments are dates, so never pass a string like FirstDayOfLastQUarter. You will need to know what Today is, since there will be references to last week, yesterday etc.",
+         "description": "This is a few shot learning example for an NL to DSL. The DSL function in question is Settlement (symbol_or_contract, datetime1). The first argument is a future (like CL, for crude oil) or a specific future contract like crude oil Dec 24 future (CLZ4). The second argument is the date at which the settlement is to be fetched. There is an overloaded function, Settlement (symbol_or_contract, begin_dt, end_dt) which gets the settlements for all dates from begin_dt to end_dt. This will also have datetimes expressed as Today, Yesterday, Last Friday, First Friday of the month, Last Thursday of August etc. You will need to know what Today is, since there will be referecnes to last week, yesterday etc. These examples are assuming Today is 10/11/2024, however you should use the Today's real date",
          "prompt": "What was the settlement for SP 500 Future on June 8th?",
          "examples": [
-            {
-               "natural_language": "What was the settlement for SP 500 Future on June 8th?",
-               "dsl": "Settlement(\"ES\", 6/8/2023)"
-            },
             {
                "natural_language": "Find Settlement price of the SP500 future on 04/22/2023.",
                "dsl": "Settlement(\"ES\", 4/22/2023)"
@@ -45,7 +41,7 @@
       },
       {
          "category": "VWAP",
-         "description": "This is a few shot learning example for an NL to DSL. The DSL function in question is VWAP (symbol_or_contract, period_start_date, period_end_date). The first argument is a future (like CL, for crude oil, or ES for SP 500) or a specific future contract like crude oil Dec 24 future (CLZ4) or SP 500 Dec 2024 (ESZ4). The second and third argument define the start and end of the VWAP period, chronologically. So for example, if we are calculating the 10 day VWAP on 10/15/2023, the period will be from 10/05/2023  to 10/15/2023 (to cover for 10 days going back from the given date). so the second argument will 10/05/2023 and the third argument will be 10/15/2023. The NL prompt  will also have datetimes expressed as Today, Yesterday, Last Friday, Last Sunday, First day of quarter, Today etc. You will need to translate these as dates that need to be passed to the VWAP DSL function. ",
+         "description": "This is a few shot learning example for an NL to DSL. The DSL function in question is VWAP (symbol_or_contract, perio_end_date, period_start_date). The first argument is a future (like CL, for crude oil, or ES for SP 500) or a specific future contract like crude oil Dec 24 future (CLZ4) or SP 500 Dec 2024 (ESZ4). The second and third argument define the end and start of the VWAP period, respectively. So for example, if we are calculating the 10 day VWAP on 10/15/2023, the period will be from 10/15/2023 going back to 10/05/2023. so the second argument will 10/05/2023 and the third argument will be 10/15/2023. The NL prompt  will also have datetimes expressed as Today, Yesterday, Last Friday, Last Sunday, First day of quarter, Today etc. You will need to translate these as dates that need to be passed to the VWAP DSL function. ",
          "prompt": "What was the VWAP for SP 500 Future over the past 30 days?",
          "examples": [
             {
@@ -53,39 +49,31 @@
                "dsl": "VWAP(\"ES\", 4/02/2023, 4/22/2023)"
             },
             {
-               "natural_language": "Find the 10 day VWAP of the Crude Oil Future on 10/10/2023.",
-               "dsl": "VWAP(\"CL\", 10/01/2023, 10/10/2023)"
+               "natural_language": "Find Yesterday's 10 day VWAP of the Crude Oil Future.",
+               "dsl": "VWAP(\"CL\", 10/01/2024, 10/10/2024)"
             },
             {
-               "natural_language": "Find today's 30-day VWAP of the 10 year treasury future on 9/11/2023.",
-               "dsl": "VWAP(\"ZN\", 8/11/2023, 9/11/2023)"
+               "natural_language": "Find today's 30-day VWAP of the 10 year treasury future.",
+               "dsl": "VWAP(\"ZN\", 9/11/2024, 10/11/2024)"
             },
             {
                "natural_language": "Find the 10 hour VWAP of the Natural Gas future from yesterday at 5:00 PM.",
                "dsl": "VWAP(\"NG\", 9/27/2024 7:00:00, 9/27/2024 17:00:00)"
-            },
-            {
-               "natural_language": "Find the 3 month VWAP of the SP 500 futures on 10/10/2023",
-               "dsl": "VWAP(\"ES\", 7/10/2023, 10/10/2023)"
             }
          ]
       },
       {
          "category": "FuturePriceDiff",
-         "description": "This is a few-shot learning example for an NL to DSL. The DSL function in question is FuturePriceDiff(symbol_or_contract, datetime1, datetime2). The first argument is a future (like CL, for crude oil, or ES for SP 500 and so on and so forth) or a specific future contract like crude oil Dec 24 future (CLZ4). The second and third arguments are datetimes (usually dates) that the function finds the difference of prices at. If the dates are given like \"first day of quarter\", \"first week of August\" I expect you to convert them into exact dates since the function only takes specific dates. The second argument should be the smaller datetime and the 3rd argument should be the end of the range, ie the higher one. Any prompt asking for price difference or price move should call FuturePriceDiff. If the dates you compute to pass to FuturePriceDiff are dates on which futures exchange might be closed (like 1/1, 12/25, Thanksgiving etc)  please adjust the date to the nearest viable trading date (I am not providing any function to find this date, you have to find the nearest trading date yourself), for example if the date falls on christmas day, change it to 12/24 and so on and so on and so forth, or the date happens to be a saturday or sunday, change it to Friday or Monday depending on which is closest",
-         "prompt": "What is the difference in SP 500 Future from July 10th to July 14th? (or how much did the SP 500 Future price move from July 10th to July 14th?)",
+         "description": "This is a few-shot learning example for an NL to DSL. The DSL function in question is FuturePriceDiff(symbol_or_contract, datetime1, datetime2). The first argument is a future (like CL, for crude oil, or ES for SP 500 and so on and so forth) or a specific future contract like crude oil Dec 24 future (CLZ4). The second and third arguments are datetimes (usually dates) that the function finds the difference of prices at. If the dates are given like \"first day of quarter\", \"first week of August\" I expect you to convert them into exact dates since the function only takes specific dates. The second argument should be the smaller datetime and the 3rd argument should be the end of the range, ie the higher one. Any prompt asking for price difference or price move should call FuturePriceDiff. If the dates you compute to pass to FuturePriceDiff are dates on which futures exchange might be closed (like 1/1, 12/25, Thanksgiving etc)  please adjust the date to the nearest viable trading date, for example if the date falls on christmas day, change it to 12/24 and so on and so on and so forth, or the date happens to be a saturday or sunday, change it to Friday or Monday depending on which is closest",
+         "prompt": "What is the difference in SP 500 Future from July 10th to July 14th?",
          "examples": [
             {
                "natural_language": "What is the difference in ZN price from July 10th to July 14th?",
                "dsl": "FuturePriceDiff(\"ZN\", 7/10/2024, 7/14/2024)"
             },
             {
-               "natural_language": "What is the difference in SP 500 Future price from the first day of the first quarter of 2020 to the last day of the first quarter of 2020?",
-               "dsl": "FuturePriceDiff(\"ES\", 1/1/2020, 3/31/2020)"
-            },
-            {
-               "natural_language": "How much did SP 500 Futures move in the first quarter of 2020?",
-               "dsl": "FuturePriceDiff(\"ES\", 1/1/2020, 3/31/2020)"
+               "natural_language": "What is the price difference in SP Future price from Sept 10th and Sept 5th?",
+               "dsl": "FuturePriceDiff(\"ES\", 9/5/2024, 9/10/2024)"
             },
             {
                "natural_language": "What is the price difference in Natural Gas Dec Future price from Aug 30th and Aug 20th?",
@@ -96,19 +84,19 @@
                "dsl": "FuturePriceDiff(\"ES\", 8/30/2023, 10/10/2023)"
             },
             {
-               "natural_language": "How much did  SP 500 Future move during the first quarter of last year",
+               "natural_language": "What is the price move in  SP 500 Future during teh first quarter of last year",
                "dsl": "FuturePriceDiff(\"ES\", 1/1/2023, 3/31/2023)"
             }
          ]
       },
       {
-         "category": "Contract Aggregate Volume",
+         "category": "Aggregate Volume",
          "description": "this is a few shot learning example for an NL to DSL. The DSL function in question is VolumeAgg(symbol, contract_type, datetime1, datetime2), which computes the highest volume in the date range, specified by datetime1, and datetime2. The first argument is a future symbol (like CL for crude oil, ES for SP 500 etc).  The second argument is the ContractType, which is \"Future\" for futures and \"FutureSpread\" for future spreads. the 3rd argument is the begin date and the 4th argument is the end date, between which the max volume is computed. If no future is specified use \"*\" from the first arugment.",
-         "prompt": "Which  SP 500 Future traded the most between 7/10/2024 and 7/14/2024",
+         "prompt": "Which  SP 500 Future traded the most between 7/14/2024 and 7/10/2024",
          "examples": [
             {
                "natural_language": "Which SP Future traded the most between 7/10/2024 and 7/14/2024",
-               "dsl": "VolumeAgg(\"ES\", \"Future\", 7/10/2024, 7/14/2024)"
+               "dsl": "VolumeAgg(\"ES\", \"Future\", 1/1/2024, 1/7/2024)"
             },
             {
                "natural_language": "Which Gold Spread traded the most in the first month of this year",
@@ -130,8 +118,8 @@
          "prompt": "When does the roll typically occur In  SP 500?",
          "examples": [
             {
-               "natural_language": "When does the roll typically occur In SP 500?",
-               "dsl": "FuturesRoll(\"ES\")"
+               "natural_language": "When does the roll typically occur In CL?",
+               "dsl": "FuturesRoll(\"CL\")"
             },
             {
                "natural_language": "When does the roll typically occur for 10 year treasury?",
@@ -175,8 +163,8 @@
          ]
       },
       {
-         "category": "Open Interest",
-         "description": "This is a few shot learning example for an NL to DSL.",
+         "category": "Average Volume",
+         "description": "This is a few shot learning example for an NL to DSL. For the purpose of this few shot prompts, lets assume today is 10/3/2024 ",
          "prompt": "When does the most open interest typically shift from one contract to another in SP 500 Future?",
          "examples": [
             {

src/DataStreamSuite/SharedStreams/SharedStreams.csproj

@@ -1,7 +1,7 @@
 <Project Sdk="Microsoft.NET.Sdk">
 
   <PropertyGroup>
-    <TargetFramework>net8.0</TargetFramework>
+    <TargetFramework>net7.0</TargetFramework>
     <ImplicitUsings>enable</ImplicitUsings>
     <Nullable>enable</Nullable>
   </PropertyGroup>

src/Spark.Admin.WebApp/src/Accounts/Types.fs

@@ -17,7 +17,7 @@ type Tab =
   | PositionManagementTab
   | MarginTab
 
-let venues = [| Venue.BINANCE; Venue.BinanceUs; Venue.BinanceUsdsFutures; Venue.CBOE; Venue.COINBASE; Venue.DERIBIT; Venue.OKX; Venue.GLBX; Venue.ICE; Venue.TAIFEX; Venue.TWSE; Venue.SIM; |]
+let venues = [| Venue.BINANCE; Venue.BinanceUs; Venue.BinanceUsdsFutures; Venue.CBOE; Venue.COINBASE; Venue.DERIBIT; Venue.OKX; Venue.GLBX; Venue.ICE; Venue.TAIFEX; Venue.SIM; |]
 
 let securityTypes = [| SecurityType.SPOT; SecurityType.FUTURE; SecurityType.OPTION; SecurityType.SPREAD; |]
 
@@ -74,11 +74,6 @@ type TaifexExchangeField =
   | Origin of CustomerOrFirm
   | ClearingAccount of string
 
-type TwseExchangeField =
-  | CTI of CTICode
-  | Origin of CustomerOrFirm
-  | ClearingAccount of string
-
 type OkxExchangeField =
   | CTI of CTICode
   | Origin of CustomerOrFirm

src/Spark.Admin.WebApp/src/Accounts/View.fs

@@ -800,7 +800,6 @@ let venueToString (venue:bts.api.Venue) =
     | bts.api.Venue.FTX -> "FTX"
     | bts.api.Venue.OKX -> "OKX"
     | bts.api.Venue.TAIFEX -> "TAIFEX"
-    | bts.api.Venue.TWSE -> "TWSE"
 
 let exchange_display expirationNameList =
   let s = sprintf "%O" expirationNameList.product.exchange

src/Spark.Admin.WebApp/src/Spark.Admin.WebApp.fsproj

@@ -1,7 +1,7 @@
 <Project Sdk="Microsoft.NET.Sdk">
 
   <PropertyGroup>
-    <TargetFramework>net8.0</TargetFramework>
+    <TargetFramework>net7.0</TargetFramework>
     <Version>1.0.0</Version>
     <AssemblyVersion>1.0.0</AssemblyVersion>
   </PropertyGroup>

src/client/Spark.Console/Spark.Console.csproj

@@ -2,8 +2,9 @@
 
   <PropertyGroup>
     <OutputType>Exe</OutputType>
-    <TargetFrameworks>net8.0</TargetFrameworks>
+    <TargetFrameworks>net7.0</TargetFrameworks>
     <Nullable>enable</Nullable>
+    <RuntimeIdentifiers>win10-x64;ubuntu.16.04-x64</RuntimeIdentifiers>
     <Authors>Blue Trading Systems</Authors>
     <Company>Blue Trading Systems</Company>
     <Product>Spark</Product>

src/client/spark/Modules/Trading/DayFills/DayFillsRowVM.cs

@@ -24,8 +24,7 @@ namespace Spark.Trading {
       DayFillsRowData _data;
 
       // display
-      //TickDisplaySettingsImmutable _quantity_display = Security.Defaults.quantity_display_settingsi;
-            TickDisplaySettingsImmutable _quantity_display = Security.Defaults.quantity_display_settingsi.with(tick_factors: new() { TickBase = 1000, TickUnit = 1 });
+      TickDisplaySettingsImmutable _quantity_display = Security.Defaults.quantity_display_settingsi;
       TickDisplaySettingsImmutable _theo_display = Security.Defaults.price_display_settingsi
          .with(tick_factors: new() {
             TickBase = 10000,

src/client/spark/Modules/Trading/Ladder/VM/LadderVM.cs

@@ -779,10 +779,10 @@ namespace mbts.spark {
                // is what was specificed in the extra parameter box
                (price, def.StopLimit.StopPrice) =
                   (def.StopLimit.StopPrice, price);
-            }
-            if (security.HasSynthetic) {
-               foreach (var d in OrderDataHelper.enumerate_algo_hierarchy(def).Where(d => d.CustomCase == CustomOrderDefinition.CustomOneofCase.Spreader)) {
-                  d.Spreader.ExecutionStrategy.SpreaderType = spreader_type;
+            } else if (def.Spreader is not null) {
+               var strategy = def.Spreader.ExecutionStrategy;
+               if (strategy is not null) {
+                  strategy.SpreaderType = spreader_type;
                }
             }
          }