Reversehedging fixed

.devcontainer/Dockerfile

@@ -65,7 +65,7 @@ RUN \
    && apt-get install -y cmake
 
 # Dotnet
-RUN apt-get update && apt-get install -y dotnet8
+RUN apt-get update && apt-get install -y dotnet7
 
 # Node
 ARG node_major=18

.gitignore

@@ -60,4 +60,3 @@ trace_data/
 *.Generated.cs
 *.lst
 .env.local
-.secrets

.vscode/launch.json

@@ -1,14 +1,6 @@
 {
   "version": "0.2.0",
   "configurations": [
-    {
-      "name": "Python Debugger: Current File with Arguments",
-      "type": "debugpy",
-      "request": "launch",
-      "program": "${file}",
-      "console": "integratedTerminal",
-      "args": "${command:pickArgs}"
-    },
     {
       "name": ".NET Core Attach",
       "type": "coreclr",
@@ -30,7 +22,7 @@
         "--config",
         "${workspaceFolder}/src/DataStreamSuite/DataStreamPublisher/Configurations/${input:dataStreamPublisherConfig}.json"
       ],
-      "cwd": "build/Debug/bin/net8.0",
+      "cwd": "build/Debug/bin/net7.0",
       "stopAtEntry": false,
       "console": "integratedTerminal"
     },
@@ -90,7 +82,7 @@
           "--config",
           "${workspaceFolder}/src/DataStreamSuite/DataStreamQueryAnalyzer/Configurations/DataStreamQueryAnalyzer.json"
         ],
-        "cwd": "build/Debug/bin/net8.0-windows",
+        "cwd": "build/Debug/bin/net7.0-windows",
         "env": {
           "PYTHON_EXE": "${workspaceFolder}/.venv/Scripts/python.exe"
         },
@@ -114,7 +106,7 @@
         "--config",
         "${workspaceFolder}/src/DataStreamSuite/PubSubServer/Configurations/${input:pubSubServerConfig}.json"
       ],
-      "cwd": "build/Debug/bin/net8.0",
+      "cwd": "build/Debug/bin/net7.0",
       "stopAtEntry": false,
       "console": "integratedTerminal"
     },
@@ -126,7 +118,7 @@
       "windows": {
         "preLaunchTask": "Build Spark (Debug)",
         "program": "Spark.exe",
-        "cwd": "build/Debug/bin/net8.0-windows",
+        "cwd": "build/Debug/bin/net7.0-windows",
       },
       "stopAtEntry": false,
       "console": "integratedTerminal"
@@ -138,7 +130,7 @@
       "program": "",
       "windows": {
         "program": "Spark.exe",
-        "cwd": "build/Debug/bin/net8.0-windows",
+        "cwd": "build/Debug/bin/net7.0-windows",
       },
       "stopAtEntry": false,
       "console": "integratedTerminal"
@@ -155,7 +147,7 @@
           "--sparkd",
           "${input:sparkConfigServer}"
         ],
-        "cwd": "build/Debug/bin/net8.0-windows",
+        "cwd": "build/Debug/bin/net7.0-windows",
       },
       "stopAtEntry": false,
       "console": "integratedTerminal"
@@ -168,7 +160,7 @@
       "windows": {
         "preLaunchTask": "Build Spark (Debug)",
         "program": "Spark.Console.exe",
-        "cwd": "build/Debug/bin/net8.0",
+        "cwd": "build/Debug/bin/net7.0",
         "args": [
           "test",
           "${input:sparkTest}"
@@ -184,7 +176,7 @@
       "program": "",
       "windows": {
         "program": "Spark.Console.exe",
-        "cwd": "build/Debug/bin/net8.0",
+        "cwd": "build/Debug/bin/net7.0",
         "args": [
           "test",
           "${input:sparkTest}"
@@ -334,4 +326,4 @@
       "description": "Test to run",
     },
   ]
-}
+}

CMakePresets.json

@@ -32,8 +32,8 @@
         "CMAKE_TOOLCHAIN_FILE": "${sourceDir}/vcpkg/scripts/buildsystems/vcpkg.cmake",
         "EXTERNALS_ROOT": "${sourceDir}/externals/windows",
         "CONFIG_DEFAULTS_PATH": "config/profiles/default/*.conf",
-        "DOTNET_BINARY_DIR": "${sourceDir}/build/$<CONFIG>/bin/net8.0",
-        "DOTNET_WINDOWS_BINARY_DIR": "${sourceDir}/build/$<CONFIG>/bin/net8.0-windows",
+        "DOTNET_BINARY_DIR": "${sourceDir}/build/$<CONFIG>/bin/net7.0",
+        "DOTNET_WINDOWS_BINARY_DIR": "${sourceDir}/build/$<CONFIG>/bin/net7.0-windows",
         "EXPORT_MD_CONSUMER": {
           "type": "BOOL",
           "value": "ON"
@@ -64,11 +64,6 @@
         "CMAKE_INSTALL_RPATH": ".",
         "CMAKE_INSTALL_PREFIX": "${sourceDir}/install/${presetName}",
         "CMAKE_EXE_LINKER_FLAGS": "-Wno-undef",
-        "CMAKE_TOOLCHAIN_FILE": "${sourceDir}/vcpkg/scripts/buildsystems/vcpkg.cmake",
-        "DATABENTO_USE_EXTERNAL_JSON": {
-          "type": "BOOL",
-          "value": "ON"
-        },
         "EXTERNALS_ROOT": "${sourceDir}/externals/linux",
         "BUILD_SDK": {
           "type": "BOOL",
@@ -187,7 +182,7 @@
         "CMAKE_CXX_FLAGS_RELEASE": "-fPIC -Wall -pedantic -Wextra -Werror -Wno-missing-braces -Wno-ignored-qualifiers --std=c++20 -fdiagnostics-color -fno-omit-frame-pointer -DNDEBUG -O3",
         "CMAKE_INSTALL_PREFIX": "${sourceDir}/install/${presetName}",
         "CONFIG_DEFAULTS_PATH": "config/profiles/default/*.conf",
-        "DOTNET_BINARY_DIR": "${sourceDir}/build/$<CONFIG>/bin/net8.0",
+        "DOTNET_BINARY_DIR": "${sourceDir}/build/$<CONFIG>/bin/net7.0",
         "EXPORT_MD_CONSUMER": {
           "type": "BOOL",
           "value": "ON"

compose.yaml

@@ -33,7 +33,6 @@ services:
       resources:
         limits:
           memory: "${SPARKD__MEMORY:-4gb}"
-
   sparkd-symbols:
     volumes:
       - ${SPARKD__DATA_DIR:-sparkd_data}:/home/bts/sparkdata/data
@@ -47,7 +46,7 @@ services:
     shm_size: "${SPARKD__SHM_SIZE:-512mb}"
     command: >
       sh -c "
-        echo bts | sudo -S chown -R bts:bts /home/bts/sparkdata
+        echo bts | sudo -S chown -R bts:bts /home/bts/sparkdata/data
         if [ -e "/home/bts/sparkdata/data/logging.sparkd/log" ]; then
           mv "/home/bts/sparkdata/data/logging.sparkd/log" "/home/bts/sparkdata/data/logging.sparkd/log_$(date '+%Y%m%d-%H%M')"
         fi

src/DataStreamSuite/ArithmeticExpressions/ArithmeticExpressions.csproj

@@ -1,7 +1,7 @@
 <Project Sdk="Microsoft.NET.Sdk">
 
   <PropertyGroup>
-    <TargetFramework>net8.0</TargetFramework>
+    <TargetFramework>net7.0</TargetFramework>
     <ImplicitUsings>enable</ImplicitUsings>
     <Nullable>enable</Nullable>
   </PropertyGroup>

src/DataStreamSuite/BaseUtilities/BaseUtilities.csproj

@@ -1,7 +1,7 @@
 <Project Sdk="Microsoft.NET.Sdk">
 
   <PropertyGroup>
-    <TargetFramework>net8.0</TargetFramework>
+    <TargetFramework>net7.0</TargetFramework>
     <ImplicitUsings>enable</ImplicitUsings>
     <Nullable>enable</Nullable>
   </PropertyGroup>

src/DataStreamSuite/BlueRegistry/BlueRegistry.csproj

@@ -1,7 +1,7 @@
 <Project Sdk="Microsoft.NET.Sdk">
 
   <PropertyGroup>
-    <TargetFramework>net8.0</TargetFramework>
+    <TargetFramework>net7.0</TargetFramework>
     <ImplicitUsings>enable</ImplicitUsings>
     <Nullable>enable</Nullable>
   </PropertyGroup>

src/DataStreamSuite/BooleanExpressions/BooleanExpressions.csproj

@@ -1,14 +1,14 @@
 <Project Sdk="Microsoft.NET.Sdk">
 
   <PropertyGroup>
-    <TargetFramework>net8.0</TargetFramework>
+    <TargetFramework>net7.0</TargetFramework>
     <ImplicitUsings>enable</ImplicitUsings>
     <Nullable>enable</Nullable>
   </PropertyGroup>
 
   <ItemGroup>
     <PackageReference Include="Google.Protobuf" Version="3.24.3" />
-    <PackageReference Include="YaccLexTools" Version="1.2.3" />
+    <PackageReference Include="YaccLexTools" Version="1.2.0" />
   </ItemGroup>
 
   <ItemGroup>

src/DataStreamSuite/BooleanExpressions/GplexBuffers.cs

@@ -24,6 +24,8 @@ namespace QUT.GplexBuffers
         public BufferException(string message) : base(message) { }
         public BufferException(string message, Exception innerException)
             : base(message, innerException) { }
+        protected BufferException(SerializationInfo info, StreamingContext context)
+            : base(info, context) { }
     }
 
     internal abstract class ScanBuff
@@ -58,10 +60,12 @@ namespace QUT.GplexBuffers
             return new BuildBuffer(source);
         }
 
+#if (!BYTEMODE)
         public static ScanBuff GetBuffer(Stream source, int fallbackCodePage)
         {
             return new BuildBuffer(source, fallbackCodePage);
         }
+#endif // !BYTEMODE
 #endif // !NOFILES
     }
 
@@ -349,15 +353,17 @@ namespace QUT.GplexBuffers
         {
             FileStream fStrm = (stream as FileStream);
             if (fStrm != null) FileName = fStrm.Name;
-            NextBlk = BlockReaderFactory.Get(stream);
+            NextBlk = BlockReaderFactory.Raw(stream);
         }
 
+#if (!BYTEMODE)
         public BuildBuffer(Stream stream, int fallbackCodePage)
         {
             FileStream fStrm = (stream as FileStream);
             if (fStrm != null) FileName = fStrm.Name;
             NextBlk = BlockReaderFactory.Get(stream, fallbackCodePage);
         }
+#endif
 
         /// <summary>
         /// Marks a conservative lower bound for the buffer,
@@ -428,19 +434,43 @@ namespace QUT.GplexBuffers
     //
     internal static class BlockReaderFactory
     {
+        public static BlockReader Raw(Stream stream)
+        {
+            return delegate(char[] block, int index, int number)
+            {
+                byte[] b = new byte[number];
+                int count = stream.Read(b, 0, number);
+                int i = 0;
+                int j = index;
+                for (; i < count; i++, j++)
+                    block[j] = (char)b[i];
+                return count;
+            };
+        }
 
-        public static BlockReader Get(Stream stream, int fallbackCodePage = -2)
+#if (!BYTEMODE)
+        public static BlockReader Get(Stream stream, int fallbackCodePage)
         {
             Encoding encoding;
             int preamble = Preamble(stream);
 
             if (preamble != 0)  // There is a valid BOM here!
                 encoding = Encoding.GetEncoding(preamble);
-            else if (fallbackCodePage >= 0) // Anything but "guess"
+            else if (fallbackCodePage == -1) // Fallback is "raw" bytes
+                return Raw(stream);
+            else if (fallbackCodePage != -2) // Anything but "guess"
                 encoding = Encoding.GetEncoding(fallbackCodePage);
             else // This is the "guess" option
-                encoding = GuessEncoding(stream);
-
+            {
+                int guess = new Guesser(stream).GuessCodePage();
+                stream.Seek(0, SeekOrigin.Begin);
+                if (guess == -1) // ==> this is a 7-bit file
+                    encoding = Encoding.ASCII;
+                else if (guess == 65001)
+                    encoding = Encoding.UTF8;
+                else             // ==> use the machine default
+                    encoding = Encoding.Default;
+            }
             StreamReader reader = new StreamReader(stream, encoding);
             return reader.Read;
         }
@@ -458,14 +488,6 @@ namespace QUT.GplexBuffers
             int b2 = stream.ReadByte();
             if (b0 == 0xef && b1 == 0xbb && b2 == 0xbf)
                 return 65001; // UTF8
-
-            int b3 = stream.ReadByte();
-            // Check for UTF-32 BOM (Byte Order Mark)
-            if (b0 == 0x00 && b1 == 0x00 && b2 == 0xFE && b3 == 0xFF)
-                return 12001; // UTF-32BE
-            if (b0 == 0xFF && b1 == 0xFE && b2 == 0x00 && b3 == 0x00)
-                return 12000; // UTF-32LE
-
             //
             // There is no unicode preamble, so we
             // return denoter for the machine default.
@@ -473,150 +495,7 @@ namespace QUT.GplexBuffers
             stream.Seek(0, SeekOrigin.Begin);
             return 0;
         }
-
-        private static Encoding GuessEncoding(Stream stream)
-        {
-            stream.Seek(0, SeekOrigin.Begin);
-            if (IsUtf16(stream, out Encoding encoding))
-            {
-                stream.Seek(0, SeekOrigin.Begin);
-                return encoding;
-            }
-
-            stream.Seek(0, SeekOrigin.Begin);
-            if (IsUtf8(stream))
-            {
-                stream.Seek(0, SeekOrigin.Begin);
-                return Encoding.UTF8;
-            }
-
-            stream.Seek(0, SeekOrigin.Begin);
-            if (IsAscii(stream))
-            {
-                stream.Seek(0, SeekOrigin.Begin);
-                return Encoding.ASCII;
-            }
-
-#if !NET462_OR_GREATER
-            stream.Seek(0, SeekOrigin.Begin);
-            if (IsLatin1(stream))
-            {
-                stream.Seek(0, SeekOrigin.Begin);
-                return Encoding.Latin1;
-            }
-#endif
-            stream.Seek(0, SeekOrigin.Begin);
-            return Encoding.Default;
-        }
-
-        private static bool IsUtf8(Stream stream)
-        {
-            int b;
-            while ((b = stream.ReadByte()) != -1)
-            {
-                if ((b & 0x80) == 0) // 0xxxxxxx
-                {
-                    continue;
-                }
-                else if ((b & 0xE0) == 0xC0) // 110xxxxx 10xxxxxx
-                {
-                    if (!IsValidUtf8ContinuationByte(stream)) return false;
-                }
-                else if ((b & 0xF0) == 0xE0) // 1110xxxx 10xxxxxx 10xxxxxx
-                {
-                    if (!IsValidUtf8ContinuationByte(stream) || !IsValidUtf8ContinuationByte(stream)) return false;
-                }
-                else if ((b & 0xF8) == 0xF0) // 11110xxx 10xxxxxx 10xxxxxx 10xxxxxx
-                {
-                    if (!IsValidUtf8ContinuationByte(stream) || !IsValidUtf8ContinuationByte(stream) || !IsValidUtf8ContinuationByte(stream)) return false;
-                }
-                else
-                {
-                    return false;
-                }
-            }
-            return true;
-        }
-
-        private static bool IsValidUtf8ContinuationByte(Stream stream)
-        {
-            int b = stream.ReadByte();
-            return b != -1 && (b & 0xC0) == 0x80; // 10xxxxxx
-        }
-
-        private static bool IsUtf16(Stream stream, out Encoding encoding)
-        {
-            encoding = Encoding.Default;
-
-            if (stream.Length >= 2)
-            {
-                // Read the first two bytes
-                int b0 = stream.ReadByte();
-                int b1 = stream.ReadByte();
-
-                // Check for UTF-16 BOM (Byte Order Mark)
-                if (b0 == 0xFF && b1 == 0xFE)
-                {
-                    encoding = Encoding.Unicode;
-                    return true; // UTF-16LE
-                }
-                if (b0 == 0xFE && b1 == 0xFF)
-                {
-                    encoding = Encoding.BigEndianUnicode;
-                    return true; // UTF-16BE
-                }
-
-                // Reset the stream position
-                stream.Seek(0, SeekOrigin.Begin);
-
-                // Check for common UTF-16 patterns
-                byte[] buffer = new byte[2];
-                while (stream.Read(buffer, 0, 2) == 2)
-                {
-                    if (buffer[0] == 0x00 && buffer[1] != 0x00)
-                    {
-                        encoding = Encoding.BigEndianUnicode;
-                        return true; // Likely UTF-16BE
-                    }
-                    if (buffer[0] != 0x00 && buffer[1] == 0x00)
-                    {
-                        encoding = Encoding.Unicode;
-                        return true; // Likely UTF-16LE
-                    }
-                }
-            }
-
-            return false;
-        }
-
-        private static bool IsAscii(Stream stream)
-        {
-            int b;
-            while ((b = stream.ReadByte()) != -1)
-            {
-                // Latin1 includes byte values from 0x00 to 0xFF
-                if (b > 0x7F)
-                {
-                    return false;
-                }
-            }
-            return true;
-        }
-
-        private static bool IsLatin1(Stream stream)
-        {
-            int b;
-            while ((b = stream.ReadByte()) != -1)
-            {
-                // Latin1 includes byte values from 0x00 to 0xFF
-                if (b > 0xFF)
-                {
-                    return false;
-                }
-            }
-            return true;
-        }
-
+#endif // !BYTEMODE
     }
 #endif // !NOFILES
     #endregion Buffer classes
@@ -661,6 +540,180 @@ namespace QUT.GplexBuffers
             return 0;
         }
     }
+#region guesser
+#if (!BYTEMODE)
+    // ==============================================================
+    // ============          Encoding Guesser           =============
+    // ==============================================================
+
+    /// <summary>
+    /// This class provides a simple finite state automaton that
+    /// scans the file looking for (1) valid UTF-8 byte patterns,
+    /// (2) bytes >= 0x80 which are not part of a UTF-8 sequence.
+    /// The method then guesses whether it is UTF-8 or maybe some 
+    /// local machine default encoding.  This works well for the
+    /// various Latin encodings.
+    /// </summary>
+    internal class Guesser
+    {
+        ScanBuff buffer;
+
+        public int GuessCodePage() { return Scan(); }
+
+        const int maxAccept = 10;
+        const int initial = 0;
+        const int eofNum = 0;
+        const int goStart = -1;
+        const int INITIAL = 0;
+        const int EndToken = 0;
+
+        #region user code
+        /* 
+         *  Reads the bytes of a file to determine if it is 
+         *  UTF-8 or a single-byte code page file.
+         */
+        public long utfX;
+        public long uppr;
+        #endregion user code
+
+        int state;
+        int currentStart = startState[0];
+        int code;
+
+        #region ScannerTables
+        static int[] startState = new int[] { 11, 0 };
+
+        #region CharacterMap
+        static sbyte[] map = new sbyte[256] {
+/*     '\0' */ 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 
+/*   '\x10' */ 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 
+/*   '\x20' */ 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 
+/*      '0' */ 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 
+/*      '@' */ 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 
+/*      'P' */ 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 
+/*      '`' */ 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 
+/*      'p' */ 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 
+/*   '\x80' */ 2, 2, 2, 2, 2, 2, 2, 2, 2, 2, 2, 2, 2, 2, 2, 2, 
+/*   '\x90' */ 2, 2, 2, 2, 2, 2, 2, 2, 2, 2, 2, 2, 2, 2, 2, 2, 
+/*   '\xA0' */ 2, 2, 2, 2, 2, 2, 2, 2, 2, 2, 2, 2, 2, 2, 2, 2, 
+/*   '\xB0' */ 2, 2, 2, 2, 2, 2, 2, 2, 2, 2, 2, 2, 2, 2, 2, 2, 
+/*   '\xC0' */ 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 
+/*   '\xD0' */ 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 
+/*   '\xE0' */ 3, 3, 3, 3, 3, 3, 3, 3, 3, 3, 3, 3, 3, 3, 3, 3, 
+/*   '\xF0' */ 4, 4, 4, 4, 4, 4, 4, 4, 5, 5, 5, 5, 5, 5, 5, 5 };
+        #endregion
+
+        static sbyte[][] nextState = new sbyte[][] {
+            new sbyte[] {0, 0, 0, 0, 0, 0},
+            new sbyte[] {-1, -1, 10, -1, -1, -1},
+            new sbyte[] {-1, -1, -1, -1, -1, -1},
+            new sbyte[] {-1, -1, 8, -1, -1, -1},
+            new sbyte[] {-1, -1, 5, -1, -1, -1},
+            new sbyte[] {-1, -1, 6, -1, -1, -1},
+            new sbyte[] {-1, -1, 7, -1, -1, -1},
+            null,
+            new sbyte[] {-1, -1, 9, -1, -1, -1},
+            null,
+            null,
+            new sbyte[] {-1, 1, 2, 3, 4, 2}
+        };
+
+
+        [SuppressMessage("Microsoft.Performance", "CA1810:InitializeReferenceTypeStaticFieldsInline")]
+        // Reason for suppression: cannot have self-reference in array initializer.
+        static Guesser()
+        {
+            nextState[7] = nextState[2];
+            nextState[9] = nextState[2];
+            nextState[10] = nextState[2];
+        }
+
+        int NextState()
+        {
+            if (code == ScanBuff.EndOfFile)
+                return eofNum;
+            else
+                return nextState[state][map[code]];
+        }
+        #endregion
+
+        public Guesser(System.IO.Stream file) { SetSource(file); }
+
+        public void SetSource(System.IO.Stream source)
+        {
+            this.buffer = new BuildBuffer(source);
+            code = buffer.Read();
+        }
+
+        int Scan()
+        {
+            for (; ; )
+            {
+                int next;
+                state = currentStart;
+                while ((next = NextState()) == goStart)
+                    code = buffer.Read();
+
+                state = next;
+                code = buffer.Read();
+
+                while ((next = NextState()) > eofNum)
+                {
+                    state = next;
+                    code = buffer.Read();
+                }
+                if (state <= maxAccept)
+                {
+                    #region ActionSwitch
+#pragma warning disable 162
+                    switch (state)
+                    {
+                        case eofNum:
+                            switch (currentStart)
+                            {
+                                case 11:
+                                    if (utfX == 0 && uppr == 0) return -1; /* raw ascii */
+                                    else if (uppr * 10 > utfX) return 0;   /* default code page */
+                                    else return 65001;                     /* UTF-8 encoding */
+                                    break;
+                            }
+                            return EndToken;
+                        case 1: // Recognized '{Upper128}',	Shortest string "\xC0"
+                        case 2: // Recognized '{Upper128}',	Shortest string "\x80"
+                        case 3: // Recognized '{Upper128}',	Shortest string "\xE0"
+                        case 4: // Recognized '{Upper128}',	Shortest string "\xF0"
+                            uppr++;
+                            break;
+                        case 5: // Recognized '{Utf8pfx4}{Utf8cont}',	Shortest string "\xF0\x80"
+                            uppr += 2;
+                            break;
+                        case 6: // Recognized '{Utf8pfx4}{Utf8cont}{2}',	Shortest string "\xF0\x80\x80"
+                            uppr += 3;
+                            break;
+                        case 7: // Recognized '{Utf8pfx4}{Utf8cont}{3}',	Shortest string "\xF0\x80\x80\x80"
+                            utfX += 3;
+                            break;
+                        case 8: // Recognized '{Utf8pfx3}{Utf8cont}',	Shortest string "\xE0\x80"
+                            uppr += 2;
+                            break;
+                        case 9: // Recognized '{Utf8pfx3}{Utf8cont}{2}',	Shortest string "\xE0\x80\x80"
+                            utfX += 2;
+                            break;
+                        case 10: // Recognized '{Utf8pfx2}{Utf8cont}',	Shortest string "\xC0\x80"
+                            utfX++;
+                            break;
+                        default:
+                            break;
+                    }
+#pragma warning restore 162
+                    #endregion
+                }
+            }
+        }
+    } // end class Guesser
+    
+#endif // !BYTEMODE
+#endregion
 #endif // !NOFILES
 
 // End of code copied from embedded resource

src/DataStreamSuite/Bts.ColumnExpressions/Bts.ColumnExpressions.csproj

@@ -1,7 +1,7 @@
 <Project Sdk="Microsoft.NET.Sdk">
 
   <PropertyGroup>
-    <TargetFramework>net8.0</TargetFramework>
+    <TargetFramework>net7.0</TargetFramework>
     <ImplicitUsings>enable</ImplicitUsings>
     <Nullable>enable</Nullable>
   </PropertyGroup>

src/DataStreamSuite/Bts.ColumnSortExpressions/Bts.ColumnSortExpressions.csproj

@@ -1,7 +1,7 @@
 <Project Sdk="Microsoft.NET.Sdk">
 
   <PropertyGroup>
-    <TargetFramework>net8.0</TargetFramework>
+    <TargetFramework>net7.0</TargetFramework>
     <ImplicitUsings>enable</ImplicitUsings>
     <Nullable>enable</Nullable>
   </PropertyGroup>

src/DataStreamSuite/ColumnMapExpressions/ColumnMapExpressions.csproj

@@ -1,7 +1,7 @@
 <Project Sdk="Microsoft.NET.Sdk">
 
   <PropertyGroup>
-    <TargetFramework>net8.0</TargetFramework>
+    <TargetFramework>net7.0</TargetFramework>
     <ImplicitUsings>enable</ImplicitUsings>
     <Nullable>enable</Nullable>
   </PropertyGroup>

src/DataStreamSuite/DataStream/DataStream.csproj

@@ -1,7 +1,7 @@
 <Project Sdk="Microsoft.NET.Sdk">
 
   <PropertyGroup>
-    <TargetFramework>net8.0</TargetFramework>
+    <TargetFramework>net7.0</TargetFramework>
     <Nullable>enable</Nullable>
   </PropertyGroup>
 
@@ -23,7 +23,6 @@
     <PackageReference Include="MongoDB.Driver" Version="2.23.1" />
     <PackageReference Include="Parquet.Net" Version="5.0.1" />
     <PackageReference Include="System.Data.SQLite" Version="1.0.118" />
-    <PackageReference Include="MathNet.Numerics" Version="5.0.0" />
   </ItemGroup>
 
 </Project>

src/DataStreamSuite/DataStream/Primitives/GroupBy/GroupByBundle.cs

@@ -4,46 +4,42 @@ namespace bts.SilverPine {
    using System;
    using System.Collections.Generic;
    using System.Linq;
-   //using MathNet.Numerics.Statistics;
 
+   using ZstdSharp.Unsafe;
 
-   public enum MultiRowOp {
+   public enum GroupByBundleOp {
       Mean,
       Median,
       Mode,
-      Range,
       StdDev,
       Max,
       Min,
       First,
       Last,
-      Sum,
-      EdgeDiff,
-      EdgeDiffPercentage,
-      //IsEdgeDiff,
+      Sum
    }
-   public class MultiRowOpHelper {
-      public readonly List<Tuple<string, MultiRowOp>> operations;
+   public class GroupByBundle {
+      public readonly List<Tuple<string, GroupByBundleOp>> operations;
 
-      public static MultiRowOpHelper parse(string text) {
+      public static GroupByBundle parse(string text) {
          // text is inp_col: op, inp_col: op, inp_col:op
-         List<Tuple<string, MultiRowOp>> rv = new List<Tuple<string, MultiRowOp>>();
+         List<Tuple<string, GroupByBundleOp>> rv = new List<Tuple<string, GroupByBundleOp>>();
          string[] calcs = text.Split(',');
          foreach (string c in calcs) {
             string[] toks = c.Split(':');
             string inp_col = toks[0].Trim();
             string op_txt = toks[1].Trim();
-            MultiRowOp op = Enum.Parse<MultiRowOp>(op_txt, true);
+            GroupByBundleOp op = Enum.Parse<GroupByBundleOp>(op_txt, true);
             rv.Add(Tuple.Create(inp_col, op));
          }
-         return new MultiRowOpHelper(rv);
+         return new GroupByBundle(rv);
       }
 
-      public MultiRowOpHelper(List<Tuple<string, MultiRowOp>> operations_) {
+      public GroupByBundle(List<Tuple<string, GroupByBundleOp>> operations_) {
          operations = operations_;
       }
 
-      public static IBlueRow group_by(List<IBlueRow> rows, MultiRowOpHelper parameters) {
+      public static IBlueRow group_by(List<IBlueRow> rows, GroupByBundle parameters) {
          if (rows.Count() == 0) {
             return null;
          }
@@ -51,8 +47,8 @@ namespace bts.SilverPine {
          IBlueRow ir = rows.ElementAt(0);
          IBlueTableSchema in_schema = ir.schema;
          IBlueTableSchema out_schema = in_schema;
-         IDictionary<string, Tuple<string, MultiRowOp>> out_col_map =
-            new Dictionary<string, Tuple<string, MultiRowOp>>();
+         IDictionary<string, Tuple<string, GroupByBundleOp>> out_col_map =
+            new Dictionary<string, Tuple<string, GroupByBundleOp>>();
          foreach (var t in parameters.operations) {
             string out_col_name = $"{t.Item1}_{t.Item2}".ToLower();
             out_schema = out_schema.add_column(TypeCode.Double, out_col_name);
@@ -62,7 +58,7 @@ namespace bts.SilverPine {
          foreach (var kv in out_col_map) {
             string out_col_name = kv.Key;
             string in_col_name = kv.Value.Item1;
-            MultiRowOp op = kv.Value.Item2;
+            GroupByBundleOp op = kv.Value.Item2;
             IEnumerable<double> col_vals = BlueDataUtility.get_colvalues<double>(rows, in_col_name);
             double comp_res = compute_op(col_vals, op);
             or.setValue(out_col_name, comp_res);
@@ -70,51 +66,20 @@ namespace bts.SilverPine {
          return or;
       }
 
-      // rolling_rows are the rolling window of rows. the 0th row is the one thats gettin
-      // calculated. rows 1 - n-1 are its window rows. the schema is based on the operations.
-
-      public IBlueRow rolling_calc(List<IBlueRow> rolling_rows) {
-         if (rolling_rows.Count() == 0) {
-            return null;
-         }
-         IBlueRow ir = rolling_rows.ElementAt(0);
-         IBlueTableSchema in_schema = ir.schema;
-         IBlueTableSchema out_schema = in_schema;
-         IDictionary<string, Tuple<string, MultiRowOp>> out_col_map =
-            new Dictionary<string, Tuple<string, MultiRowOp>>();
-         foreach (var t in operations) {
-            string out_col_name = $"{t.Item1}_{t.Item2}".ToLower();
-            out_schema = out_schema.add_column(TypeCode.Double, out_col_name);
-            out_col_map[out_col_name] = t;
-         }
-         MutableBlueRow or = new MutableBlueRow(out_schema, ir);
-         foreach (var kv in out_col_map) {
-            string out_col_name = kv.Key;
-            string in_col_name = kv.Value.Item1;
-            MultiRowOp op = kv.Value.Item2;
-            IEnumerable<double> col_vals = BlueDataUtility.get_colvalues<double>(rolling_rows, in_col_name);
-            double comp_res = compute_op(col_vals, op);
-            or.setValue(out_col_name, comp_res);
-         }
-         return or;
-      }
-
-      public static double compute_op(IEnumerable<double> col_vals, MultiRowOp op) {
+      public static double compute_op(IEnumerable<double> col_vals, GroupByBundleOp op) {
          switch (op) {
-            case MultiRowOp.Max:
+            case GroupByBundleOp.Max:
                return col_vals.Max<double>();
-            case MultiRowOp.Min:
+            case GroupByBundleOp.Min:
                return col_vals.Min<double>();
-            case MultiRowOp.Mean:
+            case GroupByBundleOp.Mean:
                return col_vals.Sum() / col_vals.Count();
-            case MultiRowOp.Sum:
+            case GroupByBundleOp.Sum:
                return col_vals.Sum();
-            case MultiRowOp.First:
+            case GroupByBundleOp.First:
                return col_vals.First();
-            case MultiRowOp.Last:
+            case GroupByBundleOp.Last:
                return col_vals.Last();
-            case MultiRowOp.StdDev:
-               return MathNet.Numerics.Statistics.Statistics.StandardDeviation(col_vals);
             default:
                // TODO
                Log.assert(false);

src/DataStreamSuite/DataStream/Primitives/GroupBy/NonIncremental/AggregationSteam.cs

@@ -1,20 +0,0 @@
-namespace bts.SilverPine {
-   using BCG;
-
-   using System;
-
-   [DataStreamUserRespresentation("AggregationStream")]
-   public class AgregationStream : ViewStream {
-
-      public AgregationStream(ViewStream parent_str, string operation_txt) {
-         ViewStream rs = new NonIncGroupByStream(parent_str, String.Empty, operation_txt);
-         mySetRealSource(rs);
-      }
-
-      public AgregationStream(ViewStream parent_str, string groupby_col, string operation_txt) {
-         ViewStream rs = new NonIncGroupByStream(parent_str, groupby_col, operation_txt);
-         mySetRealSource(rs);
-      }
-   }
-
-}

src/DataStreamSuite/DataStream/Primitives/GroupBy/NonIncremental/NonIncGroupByStream.cs

@@ -8,7 +8,7 @@ using bts;
 using bts.SilverPine;
 
 namespace BCG {
-   [DataStreamUserRespresentation("AggregateStream")]
+   [DataStreamUserRespresentation("aggregate")]
    public class NonIncGroupByStream : ViewStream {
       private class DelNonIncGroupByEvaluator : INonIncGroupByEvaluator {
          Func<string[]> _groupby_cols_func;
@@ -53,18 +53,15 @@ namespace BCG {
                                  Func<List<IBlueRow>, List<IBlueRow>> evaluate_func)
                                  : this(main_stream, new DelNonIncGroupByEvaluator(groupby_cols_func, evaluate_func)) { }
 
-      public NonIncGroupByStream(ViewStream parent_str, string groupby_col, string standard_group_by_text)
-      : this(parent_str, groupby_col, standard_group_by_text, String.Empty) { }
-
-      public NonIncGroupByStream(ViewStream parent_str, string groupby_col, string standard_group_by_text, string optional_sort_col) {
-         MultiRowOpHelper std_group_by = MultiRowOpHelper.parse(standard_group_by_text);
+      public NonIncGroupByStream(ViewStream parent_str, string groupby_col, string standard_group_by_text, string optional_sort_col = null) {
+         GroupByBundle std_group_by = GroupByBundle.parse(standard_group_by_text);
          Func<string[]> groupby_cols_func = () => {
             return String.IsNullOrEmpty(groupby_col) ? null : new string[] { groupby_col };
          };
          Func<List<IBlueRow>, List<IBlueRow>> evaluate_func = (group_rows) => {
             if (!String.IsNullOrEmpty(optional_sort_col))
                group_rows.Sort((r1, r2) => BlueDataUtility.compare_rows(r2, r1, optional_sort_col));
-            IBlueRow rv = MultiRowOpHelper.group_by(group_rows, std_group_by);
+            IBlueRow rv = GroupByBundle.group_by(group_rows, std_group_by);
             return new IBlueRow[] { rv }.ToList();
          };
          ViewStream rs = new NonIncGroupByStream(parent_str, groupby_cols_func, evaluate_func);

src/DataStreamSuite/DataStream/Primitives/Map/GlobalMapStream.cs

@@ -69,6 +69,4 @@ namespace bts {
          myFireEvent(dp_out);
       }
    }
-
-
 }

src/DataStreamSuite/DataStream/Primitives/Map/MapStream.cs

@@ -13,8 +13,9 @@ namespace BCG {
    /// desired row ie newRow = f(row), where f is the mapping fucntion. Note that the id 
    /// is preserved across the rows, ie row["id"] = f(row)["id"]
    /// </summary>
-   [DataStreamUserRespresentation(false)]
-   public class MapStream : MethodDrivenStream {
+   [DataStreamUserRespresentation (false)]
+   public class MapStream : MethodDrivenStream
+   {
       protected IStreamMapper myMapper;
       protected IChangeableStreamMapper myChangeableMapper = null;
       private bool myUseNewMap = Configuration.get<DataStreamConfig>().UseNewMap;

src/DataStreamSuite/DataStream/Primitives/Map/RollingComputationStream

@@ -1,14 +0,0 @@
-namespace bts.SilverPine
-{
-   using BCG;
-
-   [DataStreamUserRespresentation("RollingComputationStream")]
-   public class RollingComputationStream : ViewStream
-   {
-      private ViewStream _parent_str;
-      public RollingComputationStream(ViewStream parent_str, string operation_txt, string roll_column, uint window_sz)
-      {
-
-      }
-   }
-}

src/DataStreamSuite/DataStream/Primitives/Map/RollingComputationStream.cs

@@ -1,10 +0,0 @@
-namespace bts.SilverPine {
-   using BCG;
-
-
-   [DataStreamUserRespresentation("RollingComputationStream")]
-   public class RollingComputationStream : ViewStream {
-      public RollingComputationStream(ViewStream parent_str, string operation_txt, string roll_column, uint window_sz) {
-      }
-   }
-}

src/DataStreamSuite/DataStream/Primitives/Project/New/NewProjectStream.cs

@@ -108,7 +108,6 @@ namespace bts {
 
    }
 
-   [DataStreamUserRespresentation("Project")]
    public class NewProjectStream : ViewStream, IStreamMapper {
 
       private ViewStream _stream;

src/DataStreamSuite/DataStream/Primitives/Union/MutableAppendStream.cs

@@ -1,15 +1,18 @@
 using System;
 using System.Collections.Generic;
-
 using mbts;
 
-namespace BCG {
-   internal class MutableAppendTable : IBlueRowCollection {
-      private class MyRow : IBlueRow {
+namespace BCG
+{
+   internal class MutableAppendTable : IBlueRowCollection
+   {
+      private class MyRow : IBlueRow
+      {
          private IBlueRow myRow;
          private int myID;
 
-         public MyRow(IBlueRow r, int id) {
+         public MyRow(IBlueRow r, int id)
+         {
             myRow = r;
             myID = id;
          }
@@ -18,8 +21,10 @@ namespace BCG {
 
          public IBlueTableSchema schema { get { return myRow.schema; } }
 
-         public object this[int index] {
-            get {
+         public object this[int index]
+         {
+            get
+            {
                if (myRow.schema[index].colName == "id")
                   return myID;
                else
@@ -47,17 +52,20 @@ namespace BCG {
 
          public System.Collections.ICollection Keys { get { return myRow.Keys; } }
 
-         public void Remove(object key) { throw new Exception("The method or operation is not implemented."); }
+         public void Remove(object key) {throw new Exception("The method or operation is not implemented.");}
 
-         public System.Collections.ICollection Values {
-            get {
+         public System.Collections.ICollection Values
+         {
+            get
+            {
                object[] vals = new object[myRow.Count];
                CopyTo(vals, 0);
                return vals;
             }
          }
 
-         public object this[object key] {
+         public object this[object key]
+         {
             get { return key is string ? this[key as string] : this[(int)key]; }
             set { throw new Exception("The method or operation is not implemented."); }
          }
@@ -66,7 +74,8 @@ namespace BCG {
 
          #region ICollection Members
 
-         public void CopyTo(Array array, int index) {
+         public void CopyTo(Array array, int index)
+         {
             myRow.CopyTo(array, index);
             array.SetValue(myID, index + myRow.schema["id"].colNo);
          }
@@ -81,7 +90,8 @@ namespace BCG {
 
          #region IEnumerable Members
 
-         System.Collections.IEnumerator System.Collections.IEnumerable.GetEnumerator() { return new BlueRowEnumerator(this); }
+         System.Collections.IEnumerator System.Collections.IEnumerable.GetEnumerator()
+         { return new BlueRowEnumerator(this); }
 
          #endregion
       }
@@ -96,7 +106,8 @@ namespace BCG {
 
       public MutableAppendTable(IBlueTableSchema schema) { myRows = new BlueTable(schema); }
 
-      public IBlueRow addRow(IBlueRow r, int tableNo) {
+      public IBlueRow addRow(IBlueRow r, int tableNo)
+      {
          if (!myIDMap.ContainsKey(tableNo))
             myIDMap.Add(tableNo, new Dictionary<int, IBlueRow>());
          IBlueRow nr = new MyRow(r, myID++); // BlueDataUtility.getNewReplacedRow(r, myID++, "id");
@@ -105,17 +116,20 @@ namespace BCG {
          return nr;
       }
 
-      public IBlueRow removeRow(IBlueRow r, int tableNo) {
+      public IBlueRow removeRow(IBlueRow r, int tableNo)
+      {
          IBlueRow dr = myIDMap[tableNo][(int)r["id"]];
          myRows.deleteRow(dr);
          myIDMap[tableNo].Remove((int)r["id"]);
          return dr;
       }
 
-      public IBlueRowCollection removeTable(int tableNo) {
+      public IBlueRowCollection removeTable(int tableNo)
+      {
          if (!myIDMap.ContainsKey(tableNo))
             return null;
-         else {
+         else
+         {
             BlueTable rv = new BlueTable(myRows.schema);
             foreach (IBlueRow r in myIDMap[tableNo].Values)
                rv.addRow(r);
@@ -125,7 +139,8 @@ namespace BCG {
          }
       }
 
-      public void clear() {
+      public void clear()
+      {
          myRows.clear();
          myIDMap.Clear();
       }
@@ -136,7 +151,7 @@ namespace BCG {
 
       public IBlueTableSchema schema { get { return myRows.schema; } }
 
-      public bool containsID(int id) { return myRows.containsID(id); }
+      public bool containsID(int id) {return myRows.containsID(id);}
 
       #endregion
 
@@ -154,7 +169,7 @@ namespace BCG {
 
       #region IEnumerable Members
 
-      public System.Collections.IEnumerator GetEnumerator() { return myRows.GetEnumerator(); }
+      public System.Collections.IEnumerator GetEnumerator() {return myRows.GetEnumerator();}
 
       #endregion
    }
@@ -164,8 +179,8 @@ namespace BCG {
    /// to append (or remove) more streams at
    /// runtime
    /// </summary>
-   [DataStreamUserRespresentation("UnionStream")]
-   public class MutableAppendStream : ViewStream {
+   public class MutableAppendStream : ViewStream
+   {
       private bool mySubscriptionActive = false;
       private MutableAppendTable myTable;
       private int myMovingIndex;
@@ -173,17 +188,22 @@ namespace BCG {
       private bool _allign_col_orders = true;
 
       public MutableAppendStream(params ViewStream[] streams)
-         : this(new List<ViewStream>(streams)) { }
+         : this(new List<ViewStream>(streams))
+      { }
 
       public MutableAppendStream(bool allign_col_order, params ViewStream[] streams)
-         : this(allign_col_order, new List<ViewStream>(streams)) { }
+         : this(allign_col_order, new List<ViewStream>(streams))
+      { }
 
       public MutableAppendStream(ICollection<ViewStream> streams)
-         : this(true, streams) { }
+         : this(true, streams)
+      { }
 
-      public MutableAppendStream(bool allign_col_orders, ICollection<ViewStream> streams) {
+      public MutableAppendStream(bool allign_col_orders, ICollection<ViewStream> streams)
+      {
          _allign_col_orders = allign_col_orders;
-         foreach (ViewStream vs in streams) {
+         foreach (ViewStream vs in streams)
+         {
             myMovingIndex++;
             ViewStream pj = new ChangeTableNameStream(vs, myMovingIndex.ToString());
             if (_allign_col_orders)
@@ -192,7 +212,8 @@ namespace BCG {
          }
       }
 
-      public void addStream(ViewStream vs) {
+      public void addStream(ViewStream vs)
+      {
          myMovingIndex++;
          ///the change name is done to identify the callbacks, and the project to
          ///make all the schemas col orders the same
@@ -204,14 +225,19 @@ namespace BCG {
             pj.onChange += new ChangeDelegate(myHandleStreamChanges);
       }
 
-      public void removeStream(ViewStream vs) {
+      public void removeStream(ViewStream vs)
+      {
          int index = -1;
-         foreach (KeyValuePair<int, Pair<ViewStream, ViewStream>> p in myStreamIndex) {
-            if (p.Value.first == vs) {
+         foreach (KeyValuePair<int, Pair<ViewStream, ViewStream>> p in myStreamIndex)
+         {
+            if (p.Value.first == vs)
+            {
                index = p.Key;
-               if (mySubscriptionActive) {
+               if (mySubscriptionActive)
+               {
                   p.Value.second.onChange -= new ChangeDelegate(myHandleStreamChanges);
-                  if (myTable != null) {
+                  if (myTable != null)
+                  {
                      IBlueRowCollection delRows = myTable.removeTable(p.Key);
                      if (delRows != null)
                         myFireEvent(new DataPacket(myTable, new BlueTable(myTable.schema), delRows));
@@ -224,13 +250,15 @@ namespace BCG {
          }
       }
 
-      protected override void myOnFirstSubscriber() {
+      protected override void myOnFirstSubscriber()
+      {
          mySubscriptionActive = true;
          foreach (KeyValuePair<int, Pair<ViewStream, ViewStream>> p in myStreamIndex)
             p.Value.second.onChange += new ChangeDelegate(myHandleStreamChanges);
       }
 
-      protected override void myOnLastSubscriber() {
+      protected override void myOnLastSubscriber()
+      {
          mySubscriptionActive = false;
          foreach (KeyValuePair<int, Pair<ViewStream, ViewStream>> p in myStreamIndex)
             p.Value.second.onChange -= new ChangeDelegate(myHandleStreamChanges);
@@ -238,17 +266,20 @@ namespace BCG {
             myTable.clear();
       }
 
-      private void myHandleStreamChanges(object newValue) {
+      private void myHandleStreamChanges(object newValue)
+      {         
          DataPacket dp = newValue as DataPacket;
          myTable = myTable ?? new MutableAppendTable(new BlueTableSchema("MutableAppend", dp.schema.columnInfo));
          BlueTable adds = new BlueTable(myTable.schema),
                      dels = new BlueTable(myTable.schema);
          int index = Int32.Parse(dp.schema.tableName);
-         foreach (IBlueRow r in dp.deletedRows) {
+         foreach (IBlueRow r in dp.deletedRows)
+         {
             IBlueRow dr = myTable.removeRow(r, index);
             dels.addRow(dr);
          }
-         foreach (IBlueRow r in dp.addedRows) {
+         foreach (IBlueRow r in dp.addedRows)
+         {
             IBlueRow nr = myTable.addRow(r, index);
             adds.addRow(nr);
          }

src/DataStreamSuite/DataStreamCore/DataStreamCore.csproj

@@ -2,7 +2,7 @@
 
   <PropertyGroup>
     <AllowUnsafeBlocks>true</AllowUnsafeBlocks>
-    <TargetFramework>net8.0</TargetFramework>
+    <TargetFramework>net7.0</TargetFramework>
     <Nullable>disable</Nullable>
   </PropertyGroup>
 

src/DataStreamSuite/DataStreamExpressions/DataStreamExpressions.csproj

@@ -1,7 +1,7 @@
 <Project Sdk="Microsoft.NET.Sdk">
 
   <PropertyGroup>
-    <TargetFramework>net8.0</TargetFramework>
+    <TargetFramework>net7.0</TargetFramework>
     <ImplicitUsings>enable</ImplicitUsings>
     <Nullable>enable</Nullable>
   </PropertyGroup>

src/DataStreamSuite/DataStreamPublisher/DataStreamPublisher.csproj

@@ -2,7 +2,7 @@
 
   <PropertyGroup>
     <OutputType>Exe</OutputType>
-    <TargetFramework>net8.0</TargetFramework>
+    <TargetFramework>net7.0</TargetFramework>
     <Nullable>enable</Nullable>
     <AllowUnsafeBlocks>true</AllowUnsafeBlocks>
   </PropertyGroup>

src/DataStreamSuite/DataStreamQueryAnalyzer/DataStreamQueryAnalyzer.csproj

@@ -2,7 +2,7 @@
 
   <PropertyGroup>
     <OutputType>Exe</OutputType>
-    <TargetFramework>net8.0-windows</TargetFramework>
+    <TargetFramework>net7.0-windows</TargetFramework>
     <Nullable>enable</Nullable>
     <UseWindowsForms>true</UseWindowsForms>
   </PropertyGroup>

src/DataStreamSuite/DataStreamTests/DataStreamTests.csproj

@@ -1,7 +1,7 @@
 <Project Sdk="Microsoft.NET.Sdk">
 
   <PropertyGroup>
-    <TargetFramework>net8.0</TargetFramework>
+    <TargetFramework>net7.0</TargetFramework>
     <ImplicitUsings>enable</ImplicitUsings>
     <Nullable>enable</Nullable>
 

src/DataStreamSuite/ExpressionCore/ExpressionCore.csproj

@@ -1,7 +1,7 @@
 <Project Sdk="Microsoft.NET.Sdk">
 
   <PropertyGroup>
-    <TargetFramework>net8.0</TargetFramework>
+    <TargetFramework>net7.0</TargetFramework>
     <ImplicitUsings>enable</ImplicitUsings>
     <Nullable>enable</Nullable>
   </PropertyGroup>

src/DataStreamSuite/GuiUtilities/GuiUtilities.csproj

@@ -1,7 +1,7 @@
 <Project Sdk="Microsoft.NET.Sdk">
 
   <PropertyGroup>
-    <TargetFramework>net8.0-windows</TargetFramework>
+    <TargetFramework>net7.0-windows</TargetFramework>
     <ImplicitUsings>enable</ImplicitUsings>
     <Nullable>enable</Nullable>
     <UseWindowsForms>true</UseWindowsForms>

src/DataStreamSuite/PubSub/Client/PublishingUtilities/IPublisher.cs

@@ -6,18 +6,12 @@ namespace BCG
 	public interface IPublisher
 	{
       void publish( IPubItem pi );
-#pragma warning disable SYSLIB0011 // Type or member is obsolete
       void publish( IPubItem pi, IFormatter f );
-#pragma warning restore SYSLIB0011 // Type or member is obsolete
       void publish( IPubItem[] pi );
-#pragma warning disable SYSLIB0011 // Type or member is obsolete
       void publish( IPubItem[] pi, IFormatter f );
-#pragma warning restore SYSLIB0011 // Type or member is obsolete
    }
 
-#pragma warning disable SYSLIB0011 // Type or member is obsolete
    public interface IPublisher<T> where T : IFormatter
-#pragma warning restore SYSLIB0011 // Type or member is obsolete
    {
       void publish (IPubItem pi);
       void publish (IPubItem[] pi);

src/DataStreamSuite/PubSub/Misc/GenericFormatter.cs

@@ -5,21 +5,23 @@ using System.Runtime.Serialization.Formatters.Binary;
 
 namespace BCG
 {
-#pragma warning disable SYSLIB0011 // Type or member is obsolete
    public class GenericFormatter<F> : IGenericFormatter where F : IFormatter, new ()
    {
       IFormatter myFormatter = new F ();
 
       public T Deserialize<T> (Stream serializationStream)
       {
+         #pragma warning disable 
          return (T) myFormatter.Deserialize (serializationStream);
+         #pragma warning restore SYSLIB0011
       }
       public void Serialize<T> (Stream serializationStream, T graph)
       {
+         #pragma warning disable 
          myFormatter.Serialize (serializationStream, graph);
+         #pragma warning restore SYSLIB0011
       }
    }
 
    public class GenericBinaryFormatter : GenericFormatter<BinaryFormatter> { }
-#pragma warning restore SYSLIB0011 // Type or member is obsolete
 }

src/DataStreamSuite/PubSub/PubSub.csproj

@@ -6,7 +6,7 @@
 
   <PropertyGroup>
       <AllowUnsafeBlocks>true</AllowUnsafeBlocks>
-    <TargetFramework>net8.0</TargetFramework>
+    <TargetFramework>net7.0</TargetFramework>
     <Nullable>enable</Nullable>
   </PropertyGroup>
 

src/DataStreamSuite/PubSubServer/PubSubServer.csproj

@@ -2,7 +2,7 @@
 
   <PropertyGroup>
     <OutputType>Exe</OutputType>
-    <TargetFramework>net8.0</TargetFramework>
+    <TargetFramework>net7.0</TargetFramework>
     <Nullable>enable</Nullable>
   </PropertyGroup>
 

src/DataStreamSuite/SharedStreams/Business/MarketData/BBOStream.cs

@@ -13,6 +13,7 @@ namespace bts.SilverPine {
    using Spark.Api.MarketData;
 
    using SecurityKey = System.UInt32;
+   using System.Data.Entity.Infrastructure;
 
    public class FakeBBOStream : ViewStream, IWorker {
       private string _symbol;
@@ -104,9 +105,10 @@ namespace bts.SilverPine {
    public class BBOStream : ViewStream, IWorker {
       private readonly TimeSpan _poll_period = TimeSpan.FromMilliseconds(500);
       private DateTime _last_poll_time = DateTime.MinValue;
-      private ViewStreamDatabase? _vsd;
-      IReadOnlyDictionary<SecurityKey, BookTuple>? _cached_state = null;
-      MarketDataClient? _md_client;
+      private ViewStreamDatabase _vsd;
+      private int _id = 0;
+      IReadOnlyDictionary<SecurityKey, BookTuple> _cached_state = null;
+      MarketDataClient _md_client;
       internal static IBlueTableSchema _bbo_schema = BlueDataUtility.form_schema("bbo",
                                                 Tuple.Create("id", TypeCode.Int32),
                                                 Tuple.Create("sec_key", TypeCode.UInt32),
@@ -130,8 +132,9 @@ namespace bts.SilverPine {
       }
 
       protected override void myOnFirstSubscriber() {
-         _vsd?.addWorker(this);
-         _md_client?.books().Subscribe(_on_books);
+         _vsd.addWorker(this);
+         _md_client.books().Subscribe(_on_books);
+
       }
 
       private void _log_updates(TableUpdate<BookTuple, uint> update) {
@@ -153,7 +156,7 @@ namespace bts.SilverPine {
       }
 
       protected override void myOnLastSubscriber() {
-         _vsd?.removeWorker(this);
+         _vsd.removeWorker(this);
       }
 
       private void _scan_bbo() {
@@ -171,18 +174,18 @@ namespace bts.SilverPine {
                                      bt.book.asks[0].price, bt.book.asks[0].size);
             new_state.addRow(r);
          }
-         var dp = accumulate(myLastValue as DataPacket, new_state);
+         DataPacket dp = null;
+         if (myLastValue == null) {
+            dp = new DataPacket(new_state, new_state, new BlueTable(_bbo_schema));
+         } else {
+            DataPacket old_dp = myLastValue as DataPacket;
+            IBlueRowCollection added_rows, del_rows;
+            BlueDataUtility.compute_differences_fast(new_state, old_dp.newRows, out del_rows, out added_rows);
+            dp = new DataPacket(new_state, added_rows, del_rows);
+         }
          if (dp.addedRows.Count == 0 && dp.deletedRows.Count == 0)
             return;
          myFireEvent(dp);
-
-         static DataPacket accumulate(DataPacket? old_dp, BlueTable new_state) {
-            if (old_dp is null) {
-               return new(new_state, new_state, new BlueTable(_bbo_schema));
-            }
-            BlueDataUtility.compute_differences_fast(new_state, old_dp.newRows, out var del_rows, out var added_rows);
-            return new(new_state, added_rows, del_rows);
-         }
       }
 
       public void doWork() {
@@ -192,4 +195,4 @@ namespace bts.SilverPine {
          }
       }
    }
-}
+}

src/DataStreamSuite/SharedStreams/Business/MarketData/Bars/VolumeAggregationStream.cs

@@ -43,11 +43,4 @@ namespace bts.SilverPine {
          mySetRealSource(r);
       }
    }
-
-   [DataStreamUserRespresentation("PeriodicVolumeAgg")]
-   public class PeriodicVolumeAggregationStream : ViewStream {
-      public PeriodicVolumeAggregationStream(string future_contract, string contract_type, DateTime range_beg, DateTime range_end,
-                                             TimeSpan period, ViewStreamDatabase db) {
-      }
-   }
 }

src/DataStreamSuite/SharedStreams/Business/MarketData/BookStream.cs

@@ -21,7 +21,7 @@ namespace BCG {
       private DateTime _last_poll_time = DateTime.MinValue;
       private ViewStreamDatabase _vsd;
       private int _id = 0;
-      IReadOnlyDictionary<SecurityKey, BookTuple>? _cached_state = null;
+      IReadOnlyDictionary<SecurityKey, BookTuple> _cached_state = null;
       MarketDataClient _md_client;
       readonly Stopwatch _timer = new();
       private readonly IBlueTableSchema _bbo_schema = BlueDataUtility.form_schema("books",
@@ -73,8 +73,14 @@ namespace BCG {
 
       private Dictionary<SecurityKey, Dictionary<int, int>> _id_map = new Dictionary<SecurityKey, Dictionary<int, int>>();
       private int _get_id(SecurityKey skey, int depth) {
-         var d_map = _id_map.GetOrAdd(skey, _ => new() { {depth, _id++} });
-         if (!d_map.TryGetValue(depth, out int rv)) {
+         Dictionary<int, int> d_map = null;
+         if (!_id_map.TryGetValue(skey, out d_map)) {
+            d_map = new Dictionary<int, int>();
+            d_map.Add(depth, _id++);
+            _id_map.Add(skey, d_map);
+         }
+         int rv = 0;
+         if (!d_map.TryGetValue(depth, out rv)) {
             rv = _id++;
             d_map.Add(depth, rv);
          }

src/DataStreamSuite/SharedStreams/Business/MarketData/Charting/VWAPStream.cs

@@ -100,9 +100,9 @@ namespace bts.SilverPine {
       this(symbol_or_contract, beg_dt, end_dt, ViewStreamDatabase.instance) { }
 
       public VWAPStream(string symbol_or_contract, DateTime beg_dt, DateTime end_dt, ViewStreamDatabase db) {
-         //var fc = new ContractUtils.FutureContract(symbol_or_contract);
-         ViewStream r = new FutureHistoricalBarsStream(symbol_or_contract, beg_dt.ToString(), end_dt.ToString());
-         // since this is already filtered, we can groupBy iit without a groupBy column
+         var fc = new ContractUtils.FutureContract(symbol_or_contract);
+         ViewStream r = new FutureHistoricalBarsStream(fc, BarSize.ONE_DAY, new DateTimeRange(beg_dt, end_dt));
+
          r = new MathStream(r,
                  "vprice = volume * close");
          r = new NonIncGroupByStream(r,

src/DataStreamSuite/SharedStreams/Business/MarketData/FuturesActiveContractStream.cs

@@ -90,15 +90,20 @@ namespace bts.SilverPine {
       // the argument is symbol, like "ES" etc and not the contract itself
       public FuturesContinousContractStream(string symbol) : this(symbol, ViewStreamDatabase.instance) { }
       public FuturesContinousContractStream(string symbol, ViewStreamDatabase db) {
-         symbol = symbol.EndsWith("*") ? symbol : symbol + "*";
          ViewStream r = new FuturesHistorical1DayBarsStream(symbol);
-         r = new ContractExpanderStream(r, "symbol");
-         r = new FilterStream(r, "contract_type='Future'");
-         // finding the max volume across symbols, if we need to do this for open_interest, then join here 
-         // with the open interest stream on symbol/contract
-         r = new NonIncGroupByStream(r, "ts_event", "volume:Max");
-         r = new NewProjectStream(r, "ts_event symbol !");
-         r = new SortStream(r, "-ts_event");
+         // filtering for futures only
+         r = new FilterStream(r, r => {
+            string contract = r.get<string>("symbol");
+            var fc = new ContractUtils.FutureContract(contract);
+            return fc.future_tradable_contract != null;
+         });
+         r = new NonIncGroupByStream(r,
+                                      "ts_event",
+                                      "volume:Max");
+         r = new NewProjectStream(r,
+                                   "ts_event symbol !");
+         r = new SortStream(r,
+                             "-ts_event");
          mySetRealSource(r);
       }
    }

src/DataStreamSuite/SharedStreams/Business/MarketData/FuturesRollStream.cs

@@ -2,14 +2,15 @@ namespace bts.SilverPine {
    using bts;
    using BCG;
 
-   // it uses the volume (better to use open interest perhap) to detect the rollover, ie when a new 
-   // future exp becomes the highest volume one
+   // it uses the volume - better to use open interst perhap - to detect the rollover, is when a new 
+   // futuere exp becomes the highest volume one
    [DataStreamUserRespresentation("FuturesRoll")]
    public class FuturesRollStream : ViewStream {
       public FuturesRollStream(string future_symbol) {
-         ViewStream rs = new FuturesContinousContractStream(future_symbol);
-         rs = new ColumnEvolutionStream(rs, "ts_event", "symbol");
-         rs = new NewProjectStream(rs, "symbol ts_event:roll_date !");
+         //ViewStream ohlcv_str = new OHLCVStream(future_symbol);
+         ViewStream rs = new FuturesActiveContractStream(future_symbol);
+         rs = new ColumnEvolutionStream(rs, "ts_event", "contract");
+         rs = new NewProjectStream(rs, "ts_event:roll_date !");
          mySetRealSource(rs);
       }
    }

src/DataStreamSuite/SharedStreams/Business/MarketData/SettlementPriceStream.cs

@@ -2,21 +2,21 @@ namespace bts.SilverPine {
    using BCG;
 
    using bts;
-   // used directly in blueorb
+
    [DataStreamUserRespresentation("Settlement")]
    public class SettlementPriceStream : ViewStream {
 
-      // public SettlementPriceStream(string future_name, string date_txt)
-      // : this(future_name, date_txt, date_txt) { }
+      public SettlementPriceStream(string future_name, string date_txt)
+      : this(future_name, date_txt, date_txt) { }
 
-      // public SettlementPriceStream(string future_name, string begin_dt, string end_dt)
-      // : this(future_name, DateTime.Parse(begin_dt).Date, DateTime.Parse(end_dt).Date) { }
+      public SettlementPriceStream(string future_name, string begin_dt, string end_dt)
+      : this(future_name, DateTime.Parse(begin_dt).Date, DateTime.Parse(end_dt).Date) { }
 
-      public SettlementPriceStream(string future_contract, DateTime begin_dt)
-      : this(future_contract, begin_dt, begin_dt) { }
+      public SettlementPriceStream(string symbol_or_contract, DateTime begin_dt)
+      : this(symbol_or_contract, begin_dt, begin_dt) { }
 
-      public SettlementPriceStream(string future_contract, DateTime begin_dt, DateTime end_dt) {
-         ContractUtils.FutureContract fc = new ContractUtils.FutureContract(future_contract);
+      public SettlementPriceStream(string symbol_or_contract, DateTime begin_dt, DateTime end_dt) {
+         ContractUtils.FutureContract fc = new ContractUtils.FutureContract(symbol_or_contract);
          ViewStream rs = new FutureHistoricalBarsStream(fc, BarSize.ONE_DAY, new DateTimeRange(begin_dt, end_dt));
          rs = new NewProjectStream(rs, "symbol ts_event:date close:settlement !");
          mySetRealSource(rs);

src/DataStreamSuite/SharedStreams/NL/DSL_master_learnign_prompt.md

@@ -1,276 +0,0 @@
-# **DSL Master Learning Prompt**
-
-This prompt is designed to teach the foundational concepts of the DSL for financial data streaming and transformation. The DSL operates on "streams," which fetch, transform, and process financial market data in real-time or historical contexts. Forget everything I have taught you about this DSL until now (since there are some contradicting and breaking changes that can not be reconciled with anything you might have learned before) and re-learn from this prompt.
-
-# **Introduction to Streams**
-
-The DSL operates on "streams," its core functional units. Streams fetch tabular (from database, text files or network) or transform/operate on tabular data. They either act as **Datasources** (e.g., FutureHistoricalBars), that bootstrap the data or **Operators** (e.g., `FilterStream, ProjectStream, SingleRowMathStream, GroupByStream` etc) that transform or filter tabular data. Though the DSL can work on any tabular data, it is primarily written for streaming or static financial data
-
-# **Streams**
-
-## **FutureHistoricalBars (DataSource Stream)**
-
-The `FutureHistoricalBars` stream fetches OHLCV (Open, High, Low, Close, Volume) bars for financial futures contracts. It acts as a **data source** and provides the initial table that can be further processed by operators.
-
-#### **Stream arguments (parameters):**
-
-1. **`Future_contract` (string):** Specifies the contract to fetch data for. This can be one of the following 3 types  
-1. **Continuous contracts:** Use the base symbol (e.g., `ES` for S\&P 500 futures, CL for WTI Crude Oil Futures, GC for Gold Futures etc).  
-2. **Specific contracts:** Use a detailed symbol (e.g., `ESZ9` for December 2029 S\&P 500 futures, `GCU9`, for September 2025 Gold Future, `SIH9-J9`,  
-   For SIlver March April 2019 Silver Future Calendar spread)  
-3. **All Contracts for a symbol:** If instead of specifying individual contracts, data for all contracts for a specific symbol is needed, use the symbol name followed by *.* So `ES*` for all SP Futures and spread contracts, ZC\* for all corn contracts etc.  
-2. **`Bar_size` (string):**  The second argument defines the granularity of the bars. Accepted values:  
-   * `ONE_DAY` – Daily bars.  
-     * `ONE_MIN` – Minute bars.  
-3. **Optional:**  
-   * **`range_begin` (datetime):** Start of the desired time range.  
-   * **`range_end` (datetime):** End of the desired time range.
-
-#### **Schema:**
-
-The output of `FutureHistoricalBars` produces a table with the following schema (this is a classic OHLCV schema that you can find online easily)
-
-| id   | ts\_event             | contract | open    | high    | low     | close   | volume |
-| :--- | :-------------------- | :------- | :------ | :------ | :------ | :------ | :----- |
-| `0`  | `2024-06-08T00:00:00` | `ESZ9`   | 5001.50 | 5052.25 | 4990.75 | 5025.50 | 866000 |
-|      |                       |          |         |         |         |         |        |
-
-# **FilterStream (Operator Stream)**
-
-Filters rows of a parent stream based on conditions applied to its columns.
-
-#### **Stream arguments (parameters):**
-
-1. `parent_stream (another stream)` This stream provides the data, that the FilterStream subscribers to  
-2. `filter_condition (string)` This is a text expressing a boolean condition that is applied to each row of the parent stream, if it returns false, the row is filtered out
-
-#### **Examples:**
-
-1. **NL prompt:** Show me all the days when the open price for SP 500 Futures was greater than or equal to 5000:  
-   **DSL:** `r = FilterStream(FutureHistoricalBars("ES", "ONE_DAY"), "open >= 5000")`  
-2. **NL prompt:** Show me all the days when the close price for SP 500 Futures was less than 3000 and volume was greater than half a million   
-   **DSL:** `r = FilterStream(FutureHistoricalBars("ES", "ONE_MIN"), "close < 3000 and volume > 500000)`  
-3. **NL prompt:** Show me all the daily bars for the SP 500 Future spread ESU9-Z9 where its volume exceeded 100K  
-   **DSL:** `r = FilterStream(FutureHistoricalBars("ES*", "ONE_MIN"), "contract = ‘ESU9-Z9’ and volume >= 100000”)`  
-   
-
-**Notice** how expressions can be joined with AND and OR. These expressions can be nested with parenthesis.Also string needs to be in single quotes.
-
-# **ProjectStream (Operator Stream)**
-
-Changes the name of the columns in the parent stream. So for example, if we have to rename the close column to settlement, we can do this with ProjectStream
-
-#### **Stream arguments (parameters):**
-
-3. `parent_stream (another stream)` This stream provides the data, that the ProjectStream subscribers to  
-4. `renaming (string)` This is a text expressed in the form original\_col\_name:projected\_col\_name, original\_col\_name:projected\_col\_name, 
-
-#### **Examples:**
-
-	The following DSL renames the open column to O, high to H, low to L and close to C  
-**DSL:** `r = ProjectStream(FutureHistoricalBars("ES", "ONE_DAY"), "open:O, high:H, low:L, close:C”)`
-
-# **GroupByStream (Operator Stream)**
-
-The `GroupByStream` function is a DSL operator used to perform aggregation operations on rows of a parent stream. It works similarly to SQL’s `GROUP BY`, allowing rows to be grouped by a specific column (optional) and applying aggregation functions to other columns. 
-
----
-
-#### **Stream arguments (parameters):**
-
-1. **parent\_stream (another stream):** The source stream provides the data to be operated on  
-2. **Group By Column (string, optional):**
-
-   A column in the schema of the parent stream used to group rows. All rows with the same value in this column are aggregated into a single group. If omitted, the aggregation is performed over all rows of the parent stream.
-
-3. **Operation Text (string):** A string defining the column(s) to be aggregated and the operation(s) to apply, in the format:  
-   `<column_name: operation>, <column_name: operation>, …` Each `<column_name: operation>` pair produces a new column in the output stream. The new column name is formatted as `<column_name>_<operation>` in lowercase.
-
----
-
-#### **Valid Aggregation Operations:**
-
-* `Mean`: Arithmetic mean of the values.  
-* `Median`: Median of the values.  
-* `Mode`: Most frequently occurring value.  
-* `Range`: Difference between the max and min.  
-* `StdDev`: Standard deviation of the values.  
-* `Max`: Maximum value.  
-* `Min`: Minimum value.  
-* `First`: First value in the group.  
-* `Last`: Last value in the group.  
-* `Sum`: Sum of the values.  
-* `EdgeDiff`: Difference between the edge values, meaning the difference between a certain col belonging to the first and last rows respectively  
-* `EdgeDiffPercentage`: Same as `EdgeDiff` but expressed as percentage instead
-
----
-
-#### **Examples:**
-
-1. **NL:**   
-   Find the total SP 500 volume executed in the first week of 2021\.  
-   **DSL:**   
-   `bars = FutureHistoricalBars("ES", “ONE_DAY”, “2021-01-03T00:00:00”,“2021-01-11T00:00:00”)`   
-   `result = GroupByStream(bars,”volume_Sum”)`  
-2. **NL:**   
-   **Find the highest price and lowest price of the  WTI Crude Oil futures over the first half of 2024**  
-   **DSL:**   
-   `bars = FutureHistoricalBars("CL", “ONE_DAY”, “2021-01-03T00:00:00”,“2021-01-06T00:00:00”)`   
-   `result = GroupByStream(bars,”high:Max low:Min”)`  
-   `result = ProjectStream(result,”high_max:highest_price min_low: lowest_price”)`  
-3. **NL:**   
-   **Find the Average daily volume per contract in Natural Gas in 2023**  
-   **DSL:**   
-   `bars = FutureHistoricalBars("NG*", “ONE_DAY”, “2023-01-03T00:00:00”,“2023-31-12T23:59:59”)`   
-   `result = GroupByStream(bars, “contract”, ”volume:Mean”)`  
-   `result = ProjectStream(result,”mean_volume:average_daily_volume”)`
-
----
-
-#### **Schema Impact:**
-
-The `AggregateStream` modifies the schema of the parent stream by adding new columns for each aggregation operation. Existing columns are unaffected.
-
-# **SingleRowMathStream (Operator Stream)**
-
-The `SingleRowMathStream` is a DSL operator used to perform math operations, one row at a time, on the parent stream. 
-
----
-
-#### **Stream arguments (parameters):**
-
-1. **parent\_stream (another stream):**The source stream provides the base data  
-2. **math\_expression(string):** This is a math expression which allows the creation of one or more new columns that are the output of a math expression on the existing columns provided by parent\_stream. This can a simple math operation like new\_col \= a \+ b, or any function from C\#’s System.Math namespace, like new\_column \= Math.Log(Math.Pow(volume, 1.1))
-
-#### **Examples:**
-
-4. **NL:**   
-   Find the daily price movement in the SP 500 Futures through the month of Jan  
-   **DSL:**   
-   `bars = FutureHistoricalBars("ES", “ONE_DAY”, “2024-01-01T00:00:00”,“2024-01-31T00:00:00”)`   
-   `result = SingleRowMathStream(bars,”daily_move=close-open”)`  
-5. **NL:**   
-   Find the daily typical price of the  WTI Crude Oil futures over the first half of 2024  
-   **DSL:**   
-   `bars = FutureHistoricalBars("CL", “ONE_DAY”, “2021-01-03T00:00:00”,“2021-01-06T00:00:00”)`   
-   `result = SingleRowMathStream(bars, “typical_price = (high+low+close)/3.0”)`
-
-#### **Input Space (Scope):** The scope of this operation is a **single row.** The SingleRowMAthStream as the name signifies, operates on columns of the same row only. **No multi row operations are allowed.** For multi row operations using what follows.
-
-**Schema Impact:**
-
-# **MultiRowRollingMathStream (Operator Stream)**
-
-The `MultiRowRollingMathStream` function is a DSL operator used to perform mostly math operations on a rolling set of rows of a parent stream. It works similarly to the GroupByStream (described above) but instead of grouping rows together based on a column value, it operates on a rolling set of rows (where the rolling size is provided as an argument). 
-
----
-
-#### **Stream arguments (parameters):**
-
-1. **Parent Stream (another stream):** The source stream provides the data to be operated on  
-2. **Operation Text (string):** A string defining the column(s) to be operated and the operation(s) to apply, in the format:  
-   `<column_name: operation>, <column_name: operation>, …` Each `<column_name: operation>` pair produces a new column in the output stream. The new column name is formatted as `<column_name>_<operation>` in lowercase. This is like the GroupByStream’s operations string.   
-3. **Sort/Order Column Name(string):** This controls the ordering of the rolling rows. This argument will be in the format \-`<column_name>` to sort the rolling rows in descending order by the `<column_name>`, or \+`<column_name> in` ascending order.  
-4. **Rolling Window Size(int)(optional):** The size of the rolling window, If this argument is left out all, the operations are applied to all rows, is Global Operation instead of Rolling Window Operation (it's like the optional groupBy col in GroupByStream).
-
----
-
-#### **Valid Aggregation Operations:**
-
-* `Mean`: Arithmetic mean of the values.  
-* `Median`: Median of the values.  
-* `Mode`: Most frequently occurring value.  
-* `Range`: Difference between the max and min.  
-* `StdDev`: Standard deviation of the values.  
-* `Max`: Maximum value.  
-* `Min`: Minimum value.  
-* `First`: First value in the group.  
-* `Last`: Last value in the group.  
-* `Sum`: Sum of the values.  
-* `EdgeDiff`: Difference between the edge values, meaning the difference between a certain col belonging to the first and last rows respectively  
-* `EdgeDiffPercentage`: Same as `EdgeDiff` but expressed as percentage instead
-
----
-
-#### **Examples:**
-
-1. **NL:**   
-   **Find the 20 d MA of SP 500 Futures close prices**  
-   `bars = FutureHistoricalBars("ES", “ONE_DAY”, “2021-01-03T00:00:00”,“2021-01-11T00:00:00”)`   
-   `result = MultirowRollingMathStream(bars, “close:Mean”, “-ts_event”, 20)`  
-2. **NL:**   
-   **Find the 90d Standard Deviation of WTI Crude Oil during 2023**
-
-   **DSL:**   
-   `bars = FutureHistoricalBars("CL", “ONE_DAY”, “2023-01-01T00:00:00”,“2023-12-31T00:00:00”)`  
-   `// calculating typical_price`
-
-   `result = SingleRowMathStream(bars, “typical_price = (high+low+close)/3.0”)`   
-   `result = MultirowRollingMathStream(bars,“typical_price:StdDev”, “-ts_event”, 90)`
-
-   `result=ProjectStream(result, “stddev_typical_price:90dStdDev”)`
-
-3. **NL:**   
-   **Find the Average daily volume of Natural Gas in 2023**  
-   **DSL:**   
-   `bars = FutureHistoricalBars("NG", “ONE_DAY”, “2023-01-03T00:00:00”,“2023-31-12T23:59:59”)`  
-   `result = MultirowRollingMathStream(bars, ”volume:Mean”, “-ts_event”)`  
-   `result = ProjectStream(result,”mean_volume:average_daily_volume”)`
-
----
-
-#### **Schema Impact:**
-
-The `MultirowRollingMathStream` modifies the schema of the parent stream by adding new columns for each operation. Existing columns are unaffected.
-
-# **UnionStream (Operator Stream)**
-
-Like the SQL Union appends all the rows from 2 streams together.
-
-#### **Stream arguments (parameters):**
-
-5. `parent_stream1 (first stream)` This stream provides part of the data that will will be appended to the other part, provided by the following argument  
-6. `parent_stream2 (second stream)` This stream provides the other part of the data that will will be appended to the first part, provided by the first argument
-
-#### **Examples:**
-
-4. **NL prompt:** Show me all the bars across SP 500 Futures and Nasdaq Futures  
-   **DSL:** `r = UnionStream(FutureHistoricalBars("ES", "ONE_DAY"), FutureHistoricalBars("NQ", "ONE_DAY"))`  
-   
-
-#### **Schema Impact:**
-
-None
-
-# **DateTimeExpanderStream (Operator Stream)**
-
-DateTimExpanderStream expands one datetime column into various datetime components, like seconds in day, day of week, date etc, so new columns are added for those. The existing columns remain as is.
-
-#### **Stream arguments (parameters):**
-
-1. `parent_stream1 (base stream)` This parent stream that provides the data, including the datetime columns that will be expanded into multiple columns  
-2. `Datetime_input_column (string)` This is the column that will be expanded into multiple columns.
-
-**New Columns:**  
-`<input_dt_col>_date` (Date of the value, without the time part)
-
-* `<input_dt_col>_day,` (Day of the week)  
-* `<input_dt_col>_day_of_month` (day of the month, This is an int between 0 and 31\)  
-* `<input_dt_col>_month,` (month of the year, this is a numerical value between 1 and 12  
-* `input_dt_col>_hour,` (Hour of the day, value bw 0 and 23\)  
-* `<input_dt_col>_min",` (month part of this DateTime. Value is integer between 1 and 12).  
-* `<input_dt_col>_seconds_of_day",` (Total seconds of the day) 
-
-#### **Examples:**
-
-**DSL:** `r = DateTimeExpanderStream(FutureHistoricalBars("ES", "ONE_DAY"), “ts_event”)`   
-**Input Schema:**
-
-| id   | ts\_event | contract | open | high | low  | close | volume |
-| :--- | :-------- | :------- | :--- | :--- | :--- | :---- | :----- |
-
-**Output Schema:**
-
-| id   | ts\_event | contract | open | high | low  | close | volume |      |      |      |      |      |      |      |
-| :--- | :-------- | :------- | :--- | :--- | :--- | :---- | :----- | :--- | :--- | :--- | :--- | :--- | :--- | :--- |
-

src/DataStreamSuite/SharedStreams/NL/FewShotPrompts.json

@@ -2,13 +2,9 @@
    "prompts": [
       {
          "category": "Settlement",
-         "description": "This is a few shot learning example for an NL to DSL. The DSL function in question is Settlement (symbol_or_contract, datetime1). The first argument is a future (like CL, for crude oil, or ES for SP 500) or a specific future contract like crude oil Dec 24 future (CLZ4) or ESZ9 etc. The second argument is the date at which the settlement is to be fetched. There is an overloaded function, Settlement (symbol_or_contract, begin_dt, end_dt) which gets the settlements for a range pf dates from begin_dt to end_dt. You will need to convert statements  expressed as Today, Yesterday, Last Friday, First Friday of the month, Last Thursday of August, First day of last quarter etc to actual dates. The function Settlement's second and third arguments are dates, so never pass a string like FirstDayOfLastQUarter. You will need to know what Today is, since there will be references to last week, yesterday etc.",
+         "description": "This is a few shot learning example for an NL to DSL. The DSL function in question is Settlement (symbol_or_contract, datetime1). The first argument is a future (like CL, for crude oil) or a specific future contract like crude oil Dec 24 future (CLZ4). The second argument is the date at which the settlement is to be fetched. There is an overloaded function, Settlement (symbol_or_contract, begin_dt, end_dt) which gets the settlements for all dates from begin_dt to end_dt. This will also have datetimes expressed as Today, Yesterday, Last Friday, First Friday of the month, Last Thursday of August etc. You will need to know what Today is, since there will be referecnes to last week, yesterday etc. These examples are assuming Today is 10/11/2024, however you should use the Today's real date",
          "prompt": "What was the settlement for SP 500 Future on June 8th?",
          "examples": [
-            {
-               "natural_language": "What was the settlement for SP 500 Future on June 8th?",
-               "dsl": "Settlement(\"ES\", 6/8/2023)"
-            },
             {
                "natural_language": "Find Settlement price of the SP500 future on 04/22/2023.",
                "dsl": "Settlement(\"ES\", 4/22/2023)"
@@ -45,7 +41,7 @@
       },
       {
          "category": "VWAP",
-         "description": "This is a few shot learning example for an NL to DSL. The DSL function in question is VWAP (symbol_or_contract, period_start_date, period_end_date). The first argument is a future (like CL, for crude oil, or ES for SP 500) or a specific future contract like crude oil Dec 24 future (CLZ4) or SP 500 Dec 2024 (ESZ4). The second and third argument define the start and end of the VWAP period, chronologically. So for example, if we are calculating the 10 day VWAP on 10/15/2023, the period will be from 10/05/2023  to 10/15/2023 (to cover for 10 days going back from the given date). so the second argument will 10/05/2023 and the third argument will be 10/15/2023. The NL prompt  will also have datetimes expressed as Today, Yesterday, Last Friday, Last Sunday, First day of quarter, Today etc. You will need to translate these as dates that need to be passed to the VWAP DSL function. ",
+         "description": "This is a few shot learning example for an NL to DSL. The DSL function in question is VWAP (symbol_or_contract, perio_end_date, period_start_date). The first argument is a future (like CL, for crude oil, or ES for SP 500) or a specific future contract like crude oil Dec 24 future (CLZ4) or SP 500 Dec 2024 (ESZ4). The second and third argument define the end and start of the VWAP period, respectively. So for example, if we are calculating the 10 day VWAP on 10/15/2023, the period will be from 10/15/2023 going back to 10/05/2023. so the second argument will 10/05/2023 and the third argument will be 10/15/2023. The NL prompt  will also have datetimes expressed as Today, Yesterday, Last Friday, Last Sunday, First day of quarter, Today etc. You will need to translate these as dates that need to be passed to the VWAP DSL function. ",
          "prompt": "What was the VWAP for SP 500 Future over the past 30 days?",
          "examples": [
             {
@@ -53,39 +49,31 @@
                "dsl": "VWAP(\"ES\", 4/02/2023, 4/22/2023)"
             },
             {
-               "natural_language": "Find the 10 day VWAP of the Crude Oil Future on 10/10/2023.",
-               "dsl": "VWAP(\"CL\", 10/01/2023, 10/10/2023)"
+               "natural_language": "Find Yesterday's 10 day VWAP of the Crude Oil Future.",
+               "dsl": "VWAP(\"CL\", 10/01/2024, 10/10/2024)"
             },
             {
-               "natural_language": "Find today's 30-day VWAP of the 10 year treasury future on 9/11/2023.",
-               "dsl": "VWAP(\"ZN\", 8/11/2023, 9/11/2023)"
+               "natural_language": "Find today's 30-day VWAP of the 10 year treasury future.",
+               "dsl": "VWAP(\"ZN\", 9/11/2024, 10/11/2024)"
             },
             {
                "natural_language": "Find the 10 hour VWAP of the Natural Gas future from yesterday at 5:00 PM.",
                "dsl": "VWAP(\"NG\", 9/27/2024 7:00:00, 9/27/2024 17:00:00)"
-            },
-            {
-               "natural_language": "Find the 3 month VWAP of the SP 500 futures on 10/10/2023",
-               "dsl": "VWAP(\"ES\", 7/10/2023, 10/10/2023)"
             }
          ]
       },
       {
          "category": "FuturePriceDiff",
-         "description": "This is a few-shot learning example for an NL to DSL. The DSL function in question is FuturePriceDiff(symbol_or_contract, datetime1, datetime2). The first argument is a future (like CL, for crude oil, or ES for SP 500 and so on and so forth) or a specific future contract like crude oil Dec 24 future (CLZ4). The second and third arguments are datetimes (usually dates) that the function finds the difference of prices at. If the dates are given like \"first day of quarter\", \"first week of August\" I expect you to convert them into exact dates since the function only takes specific dates. The second argument should be the smaller datetime and the 3rd argument should be the end of the range, ie the higher one. Any prompt asking for price difference or price move should call FuturePriceDiff. If the dates you compute to pass to FuturePriceDiff are dates on which futures exchange might be closed (like 1/1, 12/25, Thanksgiving etc)  please adjust the date to the nearest viable trading date (I am not providing any function to find this date, you have to find the nearest trading date yourself), for example if the date falls on christmas day, change it to 12/24 and so on and so on and so forth, or the date happens to be a saturday or sunday, change it to Friday or Monday depending on which is closest",
-         "prompt": "What is the difference in SP 500 Future from July 10th to July 14th? (or how much did the SP 500 Future price move from July 10th to July 14th?)",
+         "description": "This is a few-shot learning example for an NL to DSL. The DSL function in question is FuturePriceDiff(symbol_or_contract, datetime1, datetime2). The first argument is a future (like CL, for crude oil, or ES for SP 500 and so on and so forth) or a specific future contract like crude oil Dec 24 future (CLZ4). The second and third arguments are datetimes (usually dates) that the function finds the difference of prices at. If the dates are given like \"first day of quarter\", \"first week of August\" I expect you to convert them into exact dates since the function only takes specific dates. The second argument should be the smaller datetime and the 3rd argument should be the end of the range, ie the higher one. Any prompt asking for price difference or price move should call FuturePriceDiff. If the dates you compute to pass to FuturePriceDiff are dates on which futures exchange might be closed (like 1/1, 12/25, Thanksgiving etc)  please adjust the date to the nearest viable trading date, for example if the date falls on christmas day, change it to 12/24 and so on and so on and so forth, or the date happens to be a saturday or sunday, change it to Friday or Monday depending on which is closest",
+         "prompt": "What is the difference in SP 500 Future from July 10th to July 14th?",
          "examples": [
             {
                "natural_language": "What is the difference in ZN price from July 10th to July 14th?",
                "dsl": "FuturePriceDiff(\"ZN\", 7/10/2024, 7/14/2024)"
             },
             {
-               "natural_language": "What is the difference in SP 500 Future price from the first day of the first quarter of 2020 to the last day of the first quarter of 2020?",
-               "dsl": "FuturePriceDiff(\"ES\", 1/1/2020, 3/31/2020)"
-            },
-            {
-               "natural_language": "How much did SP 500 Futures move in the first quarter of 2020?",
-               "dsl": "FuturePriceDiff(\"ES\", 1/1/2020, 3/31/2020)"
+               "natural_language": "What is the price difference in SP Future price from Sept 10th and Sept 5th?",
+               "dsl": "FuturePriceDiff(\"ES\", 9/5/2024, 9/10/2024)"
             },
             {
                "natural_language": "What is the price difference in Natural Gas Dec Future price from Aug 30th and Aug 20th?",
@@ -96,19 +84,19 @@
                "dsl": "FuturePriceDiff(\"ES\", 8/30/2023, 10/10/2023)"
             },
             {
-               "natural_language": "How much did  SP 500 Future move during the first quarter of last year",
+               "natural_language": "What is the price move in  SP 500 Future during teh first quarter of last year",
                "dsl": "FuturePriceDiff(\"ES\", 1/1/2023, 3/31/2023)"
             }
          ]
       },
       {
-         "category": "Contract Aggregate Volume",
+         "category": "Aggregate Volume",
          "description": "this is a few shot learning example for an NL to DSL. The DSL function in question is VolumeAgg(symbol, contract_type, datetime1, datetime2), which computes the highest volume in the date range, specified by datetime1, and datetime2. The first argument is a future symbol (like CL for crude oil, ES for SP 500 etc).  The second argument is the ContractType, which is \"Future\" for futures and \"FutureSpread\" for future spreads. the 3rd argument is the begin date and the 4th argument is the end date, between which the max volume is computed. If no future is specified use \"*\" from the first arugment.",
-         "prompt": "Which  SP 500 Future traded the most between 7/10/2024 and 7/14/2024",
+         "prompt": "Which  SP 500 Future traded the most between 7/14/2024 and 7/10/2024",
          "examples": [
             {
                "natural_language": "Which SP Future traded the most between 7/10/2024 and 7/14/2024",
-               "dsl": "VolumeAgg(\"ES\", \"Future\", 7/10/2024, 7/14/2024)"
+               "dsl": "VolumeAgg(\"ES\", \"Future\", 1/1/2024, 1/7/2024)"
             },
             {
                "natural_language": "Which Gold Spread traded the most in the first month of this year",
@@ -130,8 +118,8 @@
          "prompt": "When does the roll typically occur In  SP 500?",
          "examples": [
             {
-               "natural_language": "When does the roll typically occur In SP 500?",
-               "dsl": "FuturesRoll(\"ES\")"
+               "natural_language": "When does the roll typically occur In CL?",
+               "dsl": "FuturesRoll(\"CL\")"
             },
             {
                "natural_language": "When does the roll typically occur for 10 year treasury?",
@@ -175,8 +163,8 @@
          ]
       },
       {
-         "category": "Open Interest",
-         "description": "This is a few shot learning example for an NL to DSL.",
+         "category": "Average Volume",
+         "description": "This is a few shot learning example for an NL to DSL. For the purpose of this few shot prompts, lets assume today is 10/3/2024 ",
          "prompt": "When does the most open interest typically shift from one contract to another in SP 500 Future?",
          "examples": [
             {

src/DataStreamSuite/SharedStreams/SharedStreams.csproj

@@ -1,7 +1,7 @@
 <Project Sdk="Microsoft.NET.Sdk">
 
   <PropertyGroup>
-    <TargetFramework>net8.0</TargetFramework>
+    <TargetFramework>net7.0</TargetFramework>
     <ImplicitUsings>enable</ImplicitUsings>
     <Nullable>enable</Nullable>
   </PropertyGroup>

src/Spark.Admin.WebApp/src/Accounts/Types.fs

@@ -17,7 +17,7 @@ type Tab =
   | PositionManagementTab
   | MarginTab
 
-let venues = [| Venue.BINANCE; Venue.BinanceUs; Venue.BinanceUsdsFutures; Venue.CBOE; Venue.COINBASE; Venue.DERIBIT; Venue.OKX; Venue.GLBX; Venue.ICE; Venue.TAIFEX; Venue.TWSE; Venue.SIM; |]
+let venues = [| Venue.BINANCE; Venue.BinanceUs; Venue.BinanceUsdsFutures; Venue.CBOE; Venue.COINBASE; Venue.DERIBIT; Venue.OKX; Venue.GLBX; Venue.ICE; Venue.TAIFEX; Venue.SIM; |]
 
 let securityTypes = [| SecurityType.SPOT; SecurityType.FUTURE; SecurityType.OPTION; SecurityType.SPREAD; |]
 
@@ -74,11 +74,6 @@ type TaifexExchangeField =
   | Origin of CustomerOrFirm
   | ClearingAccount of string
 
-type TwseExchangeField =
-  | CTI of CTICode
-  | Origin of CustomerOrFirm
-  | ClearingAccount of string
-
 type OkxExchangeField =
   | CTI of CTICode
   | Origin of CustomerOrFirm

src/Spark.Admin.WebApp/src/Accounts/View.fs

@@ -800,7 +800,6 @@ let venueToString (venue:bts.api.Venue) =
     | bts.api.Venue.FTX -> "FTX"
     | bts.api.Venue.OKX -> "OKX"
     | bts.api.Venue.TAIFEX -> "TAIFEX"
-    | bts.api.Venue.TWSE -> "TWSE"
 
 let exchange_display expirationNameList =
   let s = sprintf "%O" expirationNameList.product.exchange

src/Spark.Admin.WebApp/src/Spark.Admin.WebApp.fsproj

@@ -1,7 +1,7 @@
 <Project Sdk="Microsoft.NET.Sdk">
 
   <PropertyGroup>
-    <TargetFramework>net8.0</TargetFramework>
+    <TargetFramework>net7.0</TargetFramework>
     <Version>1.0.0</Version>
     <AssemblyVersion>1.0.0</AssemblyVersion>
   </PropertyGroup>

src/client/Spark.Console/Spark.Console.csproj

@@ -2,8 +2,9 @@
 
   <PropertyGroup>
     <OutputType>Exe</OutputType>
-    <TargetFrameworks>net8.0</TargetFrameworks>
+    <TargetFrameworks>net7.0</TargetFrameworks>
     <Nullable>enable</Nullable>
+    <RuntimeIdentifiers>win10-x64;ubuntu.16.04-x64</RuntimeIdentifiers>
     <Authors>Blue Trading Systems</Authors>
     <Company>Blue Trading Systems</Company>
     <Product>Spark</Product>

src/client/spark/Modules/Trading/Ladder/VM/LadderVM.cs

@@ -779,10 +779,10 @@ namespace mbts.spark {
                // is what was specificed in the extra parameter box
                (price, def.StopLimit.StopPrice) =
                   (def.StopLimit.StopPrice, price);
-            }
-            if (security.HasSynthetic) {
-               foreach (var d in OrderDataHelper.enumerate_algo_hierarchy(def).Where(d => d.CustomCase == CustomOrderDefinition.CustomOneofCase.Spreader)) {
-                  d.Spreader.ExecutionStrategy.SpreaderType = spreader_type;
+            } else if (def.Spreader is not null) {
+               var strategy = def.Spreader.ExecutionStrategy;
+               if (strategy is not null) {
+                  strategy.SpreaderType = spreader_type;
                }
             }
          }

src/client/spark/Modules/Trading/PNL/Data/PNLData.cs

@@ -123,7 +123,7 @@ namespace bts.shell.pnl {
       public double display_sod_short_pos => qty_conversion.to_display(sod_short_pos);
 
       /// Day
-      public double day_pnl => new[] { day_realized_pnl, day_unrealized_pnl }.sum_real();
+      public double day_pnl => new[] { day_realized_pnl + day_unrealized_pnl }.sum_real();
       public double day_long_pos => day.long_pos;
       public double day_long_avg_price => day.long_avg;
       public double day_long_avg_pnl => bid_price - day.long_avg;
@@ -164,10 +164,8 @@ namespace bts.shell.pnl {
             day_short_pos.if_nan(0));
       public double total_short_avg_pnl => new[] { sod_short_avg_pnl, day_short_avg_pnl }.sum_real();
       public double total_net_pos => new[] { sod.long_pos, -sod.short_pos, day.long_pos, -day.short_pos }.Where(n => n.is_real()).Sum();
-      public double total_unrealized_pnl => total_long_pos > total_short_pos
-         ? (total_long_pos - total_short_pos) * effective_cpp * (bid_price - total_long_avg_price)
-         : (total_short_pos - total_long_pos) * effective_cpp * (total_short_avg_price - ask_price);
-      public double total_realized_pnl => (total_short_avg_price - total_long_avg_price) * effective_cpp * Math.Min(total_long_pos, total_short_pos);
+      public double total_unrealized_pnl => new[] { sod_unrealized_pnl, day_unrealized_pnl }.sum_real();
+      public double total_realized_pnl => new[] { sod_realized_pnl, day_realized_pnl }.sum_real();
       public double total_delta => sod.delta.add_real(day.delta);
       public double total_gamma => sod.gamma.add_real(day.gamma);
       public double total_vega => sod.vega.add_real(day.vega);

src/client/spark/Modules/Trading/PNL/VM/PNLRowVM.cs

@@ -27,9 +27,9 @@ namespace bts.shell.pnl {
       readonly PNLCellIndexBuilder<PNLRowData> _cell_map = new PNLCellIndexBuilder<PNLRowData>();
       readonly PNLCellEmbeddedAnimationModel<PNLRowData> _animator;
       readonly ISecurityMetadataStore _sec_meta_store;
-      readonly DelegateProperty<api.domain.TickDisplaySettingsImmutable> _quantity_display_backing;
       bool _refresh_normalized_data = true, _is_selected;
       PNLNormalizedData _normalized_data;
+      api.domain.TickDisplaySettingsImmutable? _quantity_display_backing;
 
       public PNLRowVM(
          PNLRowData data,
@@ -43,9 +43,6 @@ namespace bts.shell.pnl {
          _sec_meta_store = sec_meta_store;
          _animator = new PNLCellEmbeddedAnimationModel<PNLRowData>(default_appearances, triggers);
          _animator.on_row(data);
-         _quantity_display_backing = backing(
-            nameof(quantity_display),
-            () => security?.QuantityDisplay() ?? Security.Defaults.quantity_display_settingsi);
          pnl_price = _cell_map.add_price_double(ColumnKeys.PNLPrice, d => d.pnl_price);
          account_cell = _cell_map.add(ColumnKeys.Account, d => d.account);
          security_type_cell = _cell_map.add(ColumnKeys.SecurityType, d => d.security?.ContractCase ?? Security.ContractOneofCase.None);
@@ -323,8 +320,17 @@ namespace bts.shell.pnl {
       public PNLCellVM<PNLRowData, double> pnl_price { get; }
 
       // display
-      public api.domain.TickDisplaySettingsImmutable quantity_display =>
-         _quantity_display_backing.Value;
+      public api.domain.TickDisplaySettingsImmutable quantity_display {
+         get {
+            if (_quantity_display_backing.HasValue) {
+               return _quantity_display_backing.Value;
+            }
+            if (security.HasValue) {
+               _quantity_display_backing = _sec_meta_store.get_qty_display(security.Value);
+            }
+            return _quantity_display_backing ??= Security.Defaults.quantity_display_settingsi;
+         }
+      }
 
       public IList<mbts.ui.spark.CellAnimation> animations { get; private set; }
 
@@ -366,9 +372,9 @@ namespace bts.shell.pnl {
       }
 
       public void update(PNLRowData data) {
+         _quantity_display_backing = null;
          _refresh_normalized_data = true;
          this.data = data;
-         _quantity_display_backing.refresh();
          _cell_map.on_row(data);
          _animator.on_row(data);
          _cell_map.set_appearances(_animator.cell_appearances);

src/client/spark/Orders/OrderFactory.cs

@@ -143,10 +143,10 @@ namespace mbts.spark.utilities {
          bool is_child = false
       ) {
          var tifs = EnumHelper.get_values_except(TimeInForce.Undefined);
-         return order_type switch {
-            OrderType.Limit or OrderType.Iceberg => tifs,
-            _ => tifs.Except(TimeInForce.Ioc),
-         };
+         if (order_type != OrderType.Limit) {
+            tifs = tifs.Except(new[] { TimeInForce.Ioc });
+         }
+         return tifs;
       }
 
       public IEnumerable<string> dynamic_types() =>

src/client/spark/Spark.csproj

@@ -1,7 +1,7 @@
 <Project Sdk="Microsoft.NET.Sdk">
 
   <PropertyGroup>
-    <TargetFramework>net8.0-windows</TargetFramework>
+    <TargetFramework>net7.0-windows</TargetFramework>
     <UseWPF>true</UseWPF>
     <Authors>Blue Trading Systems</Authors>
     <Company>Blue Trading Systems</Company>
@@ -11,6 +11,7 @@
     <AllowUnsafeBlocks>true</AllowUnsafeBlocks>
     <Configurations>Debug;Release</Configurations>
     <Platforms>x64</Platforms>
+    <RuntimeIdentifiers>win10-x64</RuntimeIdentifiers>
     <PlatformTarget>x64</PlatformTarget>
     <LanguageTargets Condition="'$(VisualStudioVersion)'=='17.0'">
       C:/Program Files/Microsoft Visual Studio/2022/Community/Msbuild/Current/Bin/amd64/Microsoft.CSharp.targets

src/client/spark/view/Controls/DIYGrid/CellRender/CellRender.cs

@@ -70,7 +70,8 @@ namespace mbts.ui.controls {
             public double border_thickness => _border_thickness.Value;
 
             void _apply_animation() {
-               if (_animations.Select(a => a.background_animation).ExceptNull().TryGetFirst(out var bga)) {
+               var bga = _animations.OfType<BackgroundAnimation>().FirstOrDefault();
+               if (bga != null) {
                   _background.set(bga.brush);
                   if (bga.pen != null) {
                      _border.set(bga.pen.Brush);

src/client/spark_api/security/SecurityExtensions.cs

@@ -1,7 +1,4 @@
 namespace bts.api {
-   using bts.api.domain;
-   using bts.utils;
-
    using Spark.Api.Data;
 
    using System;
@@ -126,7 +123,6 @@ namespace bts.api {
             Security.ExchangeDataOneofCase.Okx => security.Okx.SecurityDesc,
             Security.ExchangeDataOneofCase.Cboe => security.Cboe.SecurityDesc,
             Security.ExchangeDataOneofCase.Taifex => security.Taifex.SecurityDesc,
-            Security.ExchangeDataOneofCase.Twse => security.Twse.SecurityDesc,
             _ => throw new NotImplementedException("security_desc not implemented for exchange")
          };
 
@@ -177,7 +173,6 @@ namespace bts.api {
             domain.Venue.Okx => Venue.Okx,
             domain.Venue.Cboe => Venue.Cboe,
             domain.Venue.Taifex => Venue.Taifex,
-            domain.Venue.Twse => Venue.Twse,
 
 
             domain.Venue.Unspecified => Venue.Unset,
@@ -199,24 +194,11 @@ namespace bts.api {
             Venue.Okx => domain.Venue.Okx,
             Venue.Cboe => domain.Venue.Cboe,
             Venue.Taifex => domain.Venue.Taifex,
-            Venue.Twse => domain.Venue.Twse,
 
             Venue.Unset => domain.Venue.Unspecified,
             _ => throw new NotImplementedException(),
          };
 
-      public static TickDisplaySettingsImmutable QuantityDisplay(this SecurityImmutable security) {
-         var qd = Security.Defaults.quantity_display_settings;
-         if (security.ContractSpec.QuantityInfos.TryGetFirst(out var qtyInfo)) {
-            qd.ShowDecimalPart = qtyInfo.TickBase.is_real_non_zero() && !(qtyInfo.TickUnit / qtyInfo.TickBase).is_int();
-            qd.TickFactors = new() {
-               TickBase = qtyInfo.TickBase,
-               TickUnit = qtyInfo.TickUnit
-            };
-         }
-         return new(qd);
-      }
-
       // ICE specific properties
       public static string market_description(this SecurityImmutable security) => security.HasIce ? security.Ice.MarketDescription : string.Empty;
       public static string product_name(this SecurityImmutable security) => security.HasIce ? security.Ice.ProductName : string.Empty;

src/client/spark_api/spark_api.csproj

@@ -1,6 +1,6 @@
 <Project Sdk="Microsoft.NET.Sdk">
   <PropertyGroup>
-    <TargetFramework>net8.0</TargetFramework>
+    <TargetFramework>net7.0</TargetFramework>
     <Authors>Blue Trading Systems</Authors>
     <Company>Blue Trading Systems</Company>
     <Description />

src/client/spark_api_server/spark_api_server.csproj

@@ -2,7 +2,7 @@
 
   <PropertyGroup>
     <OutputType>Exe</OutputType>
-    <TargetFramework>net8.0</TargetFramework>
+    <TargetFramework>net7.0</TargetFramework>
     <Nullable>enable</Nullable>
     <Platforms>x64</Platforms>
     <PlatformTarget>x64</PlatformTarget>

src/client/spark_core/Collections/Extensions/DictionaryExtensions.cs

@@ -213,6 +213,9 @@ namespace System.Collections.Generic {
          );
       }
 
+      public static Dictionary<T1, T2> ToDictionary<T1, T2>(this IEnumerable<(T1, T2)> source) =>
+         source.ToDictionary(tup => tup.Item1, tup => tup.Item2);
+
       /// <summary>
       /// Gets the value associated with the specified key, then removes the key if the lookup was successful.
       /// </summary>

src/client/spark_core/Collections/Extensions/EnumerableExtensions.cs

@@ -655,6 +655,13 @@ namespace System.Collections.Generic {
       public static IReadOnlyList<T> ToReadOnlyList<T>(this IEnumerable<T> enumerable) =>
          enumerable.ToList();
 
+      public static Dictionary<TKey, T> ToDictionary<T, TKey>(this IEnumerable<KeyValuePair<TKey, T>> key_values)
+         where TKey : notnull =>
+         key_values.ToDictionary(kv => kv.Key, kv => kv.Value);
+
+      public static Dictionary<TKey, T> ToDictionary<T, TKey>(this IEnumerable<KeyValuePair<TKey, T>> key_values, IEqualityComparer<TKey> comparer) where TKey : notnull =>
+         key_values.ToDictionary(kv => kv.Key, kv => kv.Value, comparer);
+
       public static Dictionary<TKey, T> ToDictionaryCrushed<T, TKey>(this IEnumerable<T> source, Func<T, TKey> key_selector) where TKey : notnull {
          var rv = new Dictionary<TKey, T>();
          foreach (T v in source)

src/client/spark_core/spark_core.csproj

@@ -1,7 +1,7 @@
 <Project Sdk="Microsoft.NET.Sdk">
 
   <PropertyGroup>
-    <TargetFramework>net8.0</TargetFramework>
+    <TargetFramework>net7.0</TargetFramework>
     <AllowUnsafeBlocks>true</AllowUnsafeBlocks>
     <Platforms>x64</Platforms>
     <PlatformTarget>x64</PlatformTarget>

src/client/spark_sdk/spark_sdk.csproj

@@ -1,9 +1,9 @@
 <Project Sdk="Microsoft.NET.Sdk">
   <PropertyGroup>
-    <TargetFramework>net8.0</TargetFramework>
+    <TargetFramework>net7.0</TargetFramework>
     <OutputType>Exe</OutputType>
     <GeneratePackageOnBuild Condition="$(Configuration) == 'Release' or $(Configuration) == 'Release'">true</GeneratePackageOnBuild>
-    <PackageOutputPath>$(OutputPath)\net8.0</PackageOutputPath>
+    <PackageOutputPath>$(OutputPath)\net7.0</PackageOutputPath>
     <IncludeSource>true</IncludeSource>
     <IncludeSymbols>true</IncludeSymbols>
     <TargetsForTfmSpecificBuildOutput>$(TargetsForTfmSpecificBuildOutput);CopyProjectReferencesToPackage</TargetsForTfmSpecificBuildOutput>
@@ -43,19 +43,19 @@
   <Target Name="CopyProjectReferencesToPackage" DependsOnTargets="ResolveReferences" Condition=" '$(SparkRoot)' != '' ">
     <ItemGroup>
       <BuildOutputInPackage Include="@(ReferenceCopyLocalPaths-&gt;WithMetadataValue('ReferenceSourceTarget', 'ProjectReference'))" />
-      <BuildOutputInPackage Include="$(OutputPath)..\net8.0\libspark.dll" />
+      <BuildOutputInPackage Include="$(OutputPath)..\net7.0\libspark.dll" />
     </ItemGroup>
   </Target>
 
-  <!-- deployed sdk build net8.0 -->
-  <ItemGroup Condition=" '$(SparkRoot)' == '' and '$(TargetFramework)' == 'net8.0'">
+  <!-- deployed sdk build net7.0 -->
+  <ItemGroup Condition=" '$(SparkRoot)' == '' and '$(TargetFramework)' == 'net7.0'">
     <Reference Include="spark_api">
-      <HintPath>..\..\lib\net8.0\spark_api.dll</HintPath>
+      <HintPath>..\..\lib\net7.0\spark_api.dll</HintPath>
     </Reference>
   </ItemGroup>
 
    <Target Name="copy_lib_deployed" AfterTargets="PostBuildEvent" Condition=" '$(SparkRoot)' == '' ">
-     <Copy SourceFiles="..\..\lib\net8.0\libspark.dll" DestinationFolder="$(OutputPath)\" SkipUnchangedFiles="true" />
+     <Copy SourceFiles="..\..\lib\net7.0\libspark.dll" DestinationFolder="$(OutputPath)\" SkipUnchangedFiles="true" />
    </Target>
 
 </Project>

src/client/spark_test/spark_test.csproj

@@ -1,7 +1,7 @@
 <Project Sdk="Microsoft.NET.Sdk">
 
   <PropertyGroup>
-    <TargetFramework>net8.0</TargetFramework>
+    <TargetFramework>net7.0</TargetFramework>
     <Platforms>x64</Platforms>
     <PlatformTarget>x64</PlatformTarget>
   </PropertyGroup>

src/client/spark_websocket/spark_websocket.csproj

@@ -1,7 +1,7 @@
 <Project Sdk="Microsoft.NET.Sdk">
 
   <PropertyGroup>
-    <TargetFramework>net8.0</TargetFramework>
+    <TargetFramework>net7.0</TargetFramework>
     <Nullable>enable</Nullable>
     <Platforms>x64</Platforms>
     <PlatformTarget>x64</PlatformTarget>

src/lib/api/domain/Venue.proto

@@ -27,5 +27,4 @@ enum Venue {
   VENUE_OKX = 15;
   VENUE_TAIFEX = 16;
   VENUE_CBOE = 17;
-  VENUE_TWSE = 18;
 }

src/lib/api/md/MarketDataConfig.proto

@@ -13,7 +13,6 @@ import "lib/api/md/EmumbaDeribitMarketDataConfig.proto";
 import "lib/api/md/OkxMarketDataConfig.proto";
 import "lib/api/md/GlbxMarketDataConfig.proto";
 import "lib/api/md/TaifexMarketDataConfig.proto";
-import "lib/api/md/TwseMarketDataConfig.proto";
 import "lib/api/md/CboeMarketDataConfig.proto";
 
 option csharp_namespace = "bts.api.md";
@@ -32,7 +31,6 @@ message MarketDataConfig {
   repeated GlbxMarketDataConfig glbx = 8;
   repeated CboeMarketDataConfig cboe = 9;
   repeated TaifexMarketDataConfig taifex =10;
-  repeated TwseMarketDataConfig twse = 12;
   // Add an entry here for all the exchanges that need their market data to be recorded
 
   repeated EmumbaDeribitMarketDataConfig emumba_deribit = 100;

src/lib/api/md/MarketDataConfigUtils.cpp

@@ -23,8 +23,6 @@ std::set<bts::api::Venue> MarketDataConfigUtils::get_live_exchanges(
       rv.insert(bts::api::VENUE_GLBX);
    if (cfg.taifex_size()>0)
       rv.insert(bts::api::VENUE_TAIFEX);
-   if (cfg.twse_size()>0)
-      rv.insert(bts::api::VENUE_TWSE);
    if (cfg.cboe_size()>0)
       rv.insert(bts::api::VENUE_CBOE);
    return rv;

src/lib/api/md/TwseMarketDataConfig.proto

@@ -1,19 +0,0 @@
-syntax = "proto3";
-
-import "lib/api/domain/SecurityCriteria.proto";
-
-package bts.api.md;
-
-option csharp_namespace = "bts.api.md";
-message TwseConnectionDetails{
-  string live_data_ip = 30;
-}
-
-message TwseMarketDataConfig {
-  uint32 unique_check_value = 1;
-  bool read_from_file = 20;
-  string realtime_data_file_name = 22;
-  TwseConnectionDetails connection_details = 30;
-  bts.api.domain.SecurityCriteria security_criteria = 3;
-  
-}

src/lib/api/service_helpers/AdminServiceUtils.cpp

@@ -73,7 +73,6 @@ bool AdminServiceUtils::has_exchange(
    case api::Venue::VENUE_OKX:
    case api::Venue::VENUE_BTS:
    case api::Venue::VENUE_CBOE:
-   case api::Venue::VENUE_TWSE:
    case api::Venue::VENUE_TAIFEX:
    case api::Venue::VENUE_CBOECFE:
    case api::Venue::Venue_INT_MIN_SENTINEL_DO_NOT_USE_:
@@ -190,7 +189,6 @@ AdminServiceUtils::resolve_connection(
    case api::Venue::VENUE_BINANCE_COIN_FUTURES:
    case api::Venue::VENUE_BINANCE_US:
    case api::Venue::VENUE_COINBASE:
-   case api::Venue::VENUE_TWSE:
    case api::Venue::VENUE_TAIFEX:
    case api::Venue::VENUE_CBOE:
    case api::Venue::VENUE_BINANCE_USDS_FUTURES: {
@@ -465,7 +463,6 @@ std::string AdminServiceUtils::find_account_exchange_account(
    case api::Venue::VENUE_OKX:
    case api::Venue::VENUE_BTS:
    case api::Venue::VENUE_CBOE:
-   case api::Venue::VENUE_TWSE:
    case api::Venue::VENUE_TAIFEX:
    case api::Venue::VENUE_CBOECFE:
    case api::Venue::Venue_INT_MIN_SENTINEL_DO_NOT_USE_:
@@ -790,7 +787,6 @@ bts::api::orderman::RoutingRecord AdminServiceUtils::get_routing_record(
    case api::Venue::VENUE_COINBASE:
    case api::Venue::VENUE_UNSET:
    case api::Venue::VENUE_BTS:
-   case api::Venue::VENUE_TWSE:
    case api::Venue::VENUE_CBOECFE:
    case api::Venue::Venue_INT_MIN_SENTINEL_DO_NOT_USE_:
    case api::Venue::Venue_INT_MAX_SENTINEL_DO_NOT_USE_:
@@ -1187,7 +1183,6 @@ std::string AdminServiceUtils::find_user_exchange_id(
    case api::Venue::VENUE_UNSET:
    case api::Venue::VENUE_BTS:
    case api::Venue::VENUE_CBOE:
-   case api::Venue::VENUE_TWSE:
    case api::Venue::VENUE_TAIFEX:
    case api::Venue::VENUE_CBOECFE:
    case api::Venue::Venue_INT_MIN_SENTINEL_DO_NOT_USE_:
@@ -1227,7 +1222,6 @@ std::string AdminServiceUtils::find_user_company_id(
    case api::Venue::VENUE_UNSET:
    case api::Venue::VENUE_BTS:
    case api::Venue::VENUE_CBOE:
-   case api::Venue::VENUE_TWSE:
    case api::Venue::VENUE_TAIFEX:
 
    case api::Venue::VENUE_CBOECFE:

src/lib/cmake/GetLinuxServerLinkLibraries.cmake

@@ -1,8 +1,4 @@
 function(get_link_libraries dest)
-  find_package(fmt CONFIG REQUIRED)
-  find_package(nlohmann_json CONFIG REQUIRED)
-  find_package(OpenSSL REQUIRED)
-  find_package(Catch2 CONFIG REQUIRED)
   set(${dest}
     protobuf
     double-conversion
@@ -13,16 +9,14 @@ function(get_link_libraries dest)
     boost_chrono
     boost_date_time
     boost_atomic
-    Catch2::Catch2
     SQLiteCpp
     sqlite3
     curl
     uuid
     ssl
-    fmt::fmt-header-only
-    nlohmann_json::nlohmann_json
-    OpenSSL::SSL
-    OpenSSL::Crypto
+    crypto
+    debug fmtd
+    optimized fmt
     tbb
     cryptopp
     pugixml

src/lib/collections/concurrent_factory.h

@@ -27,21 +27,6 @@ class ConcurrentFactory {
       });
    }
 
-   std::shared_ptr<const ValueT> get(
-       const KeyT &key, bool assert_exists = false) {
-      return store.lock([&](auto &map) {
-         auto it = map.find(key);
-         const bool exists = it != map.end();
-         if (assert_exists) {
-            bts_assert_msgf(
-                exists, "{}::get. Key not found '{}'",
-                utils::cached_type_name<ConcurrentFactory<KeyT, ValueT>>(),
-                key);
-         }
-         return exists ? it->second : nullptr;
-      });
-   }
-
    std::size_t erase(const KeyT &key) {
       return store.lock([&](auto &map) {
          map.erase(key);

src/lib/collections/concurrent_queue.h

@@ -1,45 +0,0 @@
-#pragma once
-
-#include <mutex>
-#include <optional>
-#include <queue>
-
-namespace bts::collections {
-
-template <class T>
-class ConcurrentQueue {
-   std::mutex mx;
-   std::queue<T> queue;
-
- public:
-   bool empty() {
-      std::lock_guard<std::mutex> lock{mx};
-      return queue.empty();
-   }
-
-   void push(T item) {
-      std::lock_guard<std::mutex> lock{mx};
-      queue.push(std::move(item));
-   }
-
-   bool pop(T &res) {
-      std::lock_guard<std::mutex> lock{mx};
-      if (queue.empty()) {
-         return false;
-      }
-      res = std::move(queue.front());
-      queue.pop();
-      return true;
-   }
-
-   std::optional<T> pop() {
-      std::lock_guard<std::mutex> lock{mx};
-      if (queue.empty()) {
-         return std::nullopt;
-      }
-      std::optional<T> rv = std::move(queue.front());
-      queue.pop();
-      return rv;
-   }
-};
-} // namespace bts::collections

src/lib/communincation/websockets/tests/WebsocketSyncClientTests.cpp

@@ -11,7 +11,7 @@
 #include <boost/asio/ip/tcp.hpp>
 #include <boost/asio/ssl/stream.hpp>
 
-#include <catch2/catch_all.hpp>
+#include <catch/catch.hpp>
 #include <cstdlib>
 #include <functional>
 #include <iostream>
@@ -198,4 +198,4 @@ TEST_CASE("WebsocketSyncClientTest", "[ws_ssl]") {
          }
       }
    }
-}
+}

src/lib/config/profiles/default/symbols.conf

@@ -24,14 +24,6 @@ SymbolLists {
       symbol: "BTC"
       symbol: "ETH"
     }
-    list {
-      key: "TWSE"
-      symbol: "2330"
-      symbol: "2317"
-      symbol: "2454"
-      symbol: "2603"
-      symbol: "2615"
-    }
     list {
       key: "TAIFEX"
       symbol: "QFFH"
@@ -73,11 +65,6 @@ Symbols {
       future: @SymbolLists.list[GLBX].symbol
       spread: @SymbolLists.list[GLBX].symbol
     }
-    venues{
-      name: "VENUE_TWSE"
-      future: @SymbolLists[TWSE].symbol
-      option: @SymbolLists[TWSE].symbol
-    }
     venues{
       name: "VENUE_TAIFEX"
       future: @SymbolLists[TAIFEX].symbol
@@ -110,11 +97,6 @@ Symbols {
       future: @SymbolLists.list[ICE].symbol
       spread: @SymbolLists.list[ICE].symbol
     }
-    venues{
-      name: "VENUE_TWSE"
-      future: @SymbolLists[TWSE].symbol
-      option: @SymbolLists[TWSE].symbol
-    }
     venues{
       name: "VENUE_TAIFEX"
       future: @SymbolLists[TAIFEX].symbol
@@ -149,11 +131,6 @@ Symbols {
       option: @SymbolLists.list[DERIBIT].symbol
       spread: @SymbolLists.list[DERIBIT].symbol
     }
-    venues{
-      name: "VENUE_TWSE"
-      future: @SymbolLists.list[TWSE].symbol
-      option: @SymbolLists.list[TWSE].symbol
-    }
     venues{
       name: "VENUE_TAIFEX"
       future: @SymbolLists.list[TAIFEX].symbol

src/lib/config/tests/config_ref_tests.cpp

@@ -9,8 +9,7 @@
 #include "lib/expression/ParserExceptions.h"
 
 #include "HTTPServer.h"
-
-#include <catch2/catch_all.hpp>
+#include "catch/catch.hpp"
 #include <iostream>
 #include <memory>
 

src/lib/config/tests/config_tests.cpp

@@ -19,7 +19,7 @@
 
 #define URDL_HEADER_ONLY 1
 #define URDL_DISALE_SSL 1
-#include <catch2/catch_all.hpp>
+#include <catch/catch.hpp>
 #include <urdl/test/http_server.hpp>
 #include <urdl/url.hpp>
 

src/lib/core/LockGuard.h

@@ -0,0 +1,4 @@
+#pragma once
+
+#define LOCK_GUARD(lockable)                                                   \
+   std::lock_guard<decltype(lockable)> __lock_guard_##lockable { lockable }

src/lib/core/TimeStamp.cpp

@@ -69,7 +69,7 @@ TimeStamp TimeStamp::from_ptime(boost::posix_time::ptime t) {
    auto epoch_pt =
        t - boost::posix_time::ptime(boost::gregorian::date(1970, 1, 1));
    auto epoch_nanos = epoch_pt.total_nanoseconds();
-   return TimeStamp{epoch_nanos};
+   return TimeStamp{static_cast<std::uint64_t>(epoch_nanos)};
 }
 
 } // namespace bts::core

src/lib/core/TimeStamp.h

@@ -25,7 +25,7 @@ namespace bts::core {
 struct TimeStamp {
  public:
    constexpr TimeStamp() = default;
-   explicit constexpr TimeStamp(std::int64_t value);
+   explicit constexpr TimeStamp(std::uint64_t value);
    TimeStamp(std::uint32_t year, std::uint8_t month, std::uint8_t day);
    TimeStamp(
        std::uint32_t year,
@@ -74,10 +74,10 @@ struct TimeStamp {
        std::uint8_t second,
        TimeStamp *result);
 
-   constexpr std::int64_t nanos() const;
-   constexpr std::int64_t micros() const;
-   constexpr std::int64_t millis() const;
-   constexpr std::int64_t seconds() const;
+   constexpr std::uint64_t nanos() const;
+   constexpr std::uint64_t micros() const;
+   constexpr std::uint64_t millis() const;
+   constexpr std::uint64_t seconds() const;
 
    constexpr double total_millis() const;
    constexpr double total_seconds() const;
@@ -91,7 +91,7 @@ struct TimeStamp {
    void to_protobuf(google::protobuf::Timestamp &ts) const;
    std::string to_iso_string() const;
    // current epoch ns
-   static int64_t now_ns() {
+   static uint64_t now_ns() {
 #ifndef _WIN32
       return datetime::unix_nano_now();
 #else
@@ -106,7 +106,7 @@ struct TimeStamp {
 
  private:
    // posix time
-   std::int64_t value_{};
+   std::uint64_t value_{};
 };
 
 constexpr bool operator==(TimeStamp lhs, TimeStamp rhs) noexcept;
@@ -128,7 +128,7 @@ std::ostream &operator<<(std::ostream &out, const TimeStamp &ts) noexcept;
 // Definition
 //////////////////////////////////////////////////
 
-constexpr TimeStamp::TimeStamp(std::int64_t value) : value_(value) {}
+constexpr TimeStamp::TimeStamp(std::uint64_t value) : value_(value) {}
 
 inline TimeStamp::TimeStamp(
     std::uint32_t year, std::uint8_t month, std::uint8_t day)
@@ -137,44 +137,44 @@ inline TimeStamp::TimeStamp(
 constexpr TimeStamp TimeStamp::min() { return TimeStamp{0}; }
 
 constexpr TimeStamp TimeStamp::max() {
-   return TimeStamp{std::numeric_limits<std::int64_t>::max()};
+   return TimeStamp{std::numeric_limits<std::uint64_t>::max()};
 }
 
 constexpr TimeStamp TimeStamp::from_int64(std::int64_t value) {
    bts_assert(value >= 0);
-   return TimeStamp{static_cast<std::int64_t>(value)};
+   return TimeStamp{static_cast<std::uint64_t>(value)};
 }
 
 constexpr TimeStamp TimeStamp::from_nanos(std::int64_t value) {
    bts_assert(value >= 0);
-   return TimeStamp{static_cast<std::int64_t>(value)};
+   return TimeStamp{static_cast<std::uint64_t>(value)};
 }
 
 constexpr TimeStamp TimeStamp::from_micros(std::int64_t value) {
    bts_assert(value >= 0);
    constexpr const auto nanos_per_micro = 1'000;
-   return TimeStamp{static_cast<std::int64_t>(value * nanos_per_micro)};
+   return TimeStamp{static_cast<std::uint64_t>(value * nanos_per_micro)};
 }
 
 constexpr TimeStamp TimeStamp::from_millis(std::int64_t value) {
    bts_assert(value >= 0);
    constexpr const auto nanos_per_milli = 1'000'000;
-   return TimeStamp{static_cast<std::int64_t>(value * nanos_per_milli)};
+   return TimeStamp{static_cast<std::uint64_t>(value * nanos_per_milli)};
 }
 
 constexpr TimeStamp TimeStamp::from_seconds(std::int64_t value) {
    bts_assert(value >= 0);
    constexpr const auto nanos_per_second = 1'000'000'000;
-   return TimeStamp{static_cast<std::int64_t>(value * nanos_per_second)};
+   return TimeStamp{static_cast<std::uint64_t>(value * nanos_per_second)};
 }
 
-constexpr std::int64_t TimeStamp::nanos() const { return value_; }
+constexpr std::uint64_t TimeStamp::nanos() const { return value_; }
 
-constexpr std::int64_t TimeStamp::micros() const { return value_ / 1'000; }
+constexpr std::uint64_t TimeStamp::micros() const { return value_ / 1'000; }
 
-constexpr std::int64_t TimeStamp::millis() const { return value_ / 1'000'000; }
+constexpr std::uint64_t TimeStamp::millis() const { return value_ / 1'000'000; }
 
-constexpr std::int64_t TimeStamp::seconds() const {
+constexpr std::uint64_t TimeStamp::seconds() const {
    return value_ / 1'000'000'000;
 }
 
@@ -201,7 +201,7 @@ inline std::int64_t TimeStamp::to_date_int() const {
 
 inline void TimeStamp::to_date(
     uint32_t &year, uint32_t &month, uint32_t &day) const {
-   std::int64_t epoch_secs = value_ / (1000 * 1000 * 1000);
+   std::uint64_t epoch_secs = value_ / (1000 * 1000 * 1000);
    time_t ts_tt(epoch_secs);
    struct tm *ptm = gmtime(&ts_tt);
    year = ptm->tm_year + 1900;

src/lib/datastreams/Tables/tests/stream_tests.cpp

@@ -1,12 +1,11 @@
 
+#include "catch/catch.hpp"
 #include "lib/api/service_helpers/AdminServiceUtils.h"
 #include "lib/api/services/AdminService.server.pb.h"
 #include "lib/config/config.h"
 #include "lib/datastreams/Tables/StreamData.h"
 #include "lib/utils/bts_chrono.h"
 #include "lib/utils/helper_macros.h"
-
-#include <catch2/catch_all.hpp>
 #include <memory>
 
 using namespace bts::streams;

src/lib/domain/DomainSecurityUtils.cpp

@@ -23,8 +23,6 @@ api::domain::Venue DomainSecurityUtils::convert(bts::api::Venue venue) {
       return DOMAIN_VENUE(DERIBIT);
    case bts::api::Venue::VENUE_CBOE:
       return DOMAIN_VENUE(CBOE);
-   case bts::api::Venue::VENUE_TWSE:
-      return DOMAIN_VENUE(TWSE);
    case bts::api::Venue::VENUE_TAIFEX:
       return DOMAIN_VENUE(TAIFEX);
    case bts::api::Venue::VENUE_BTS:
@@ -61,8 +59,6 @@ bts::api::Venue DomainSecurityUtils::convert(api::domain::Venue venue) {
       return api::Venue::VENUE_DERIBIT;
    case DOMAIN_VENUE(CBOE):
       return api::Venue::VENUE_CBOE;
-   case DOMAIN_VENUE(TWSE):
-      return api::Venue::VENUE_TWSE;
    case DOMAIN_VENUE(TAIFEX):
       return api::Venue::VENUE_TAIFEX;
    case DOMAIN_VENUE(FTX):
@@ -175,9 +171,6 @@ bool DomainSecurityUtils::get_security_desc(
    case bts::api::Security::ExchangeDataCase::kCboe:
       desc = security.cboe().security_desc();
       break;
-   case bts::api::Security::ExchangeDataCase::kTwse:
-      desc = security.twse().security_desc();
-      break;
    case bts::api::Security::ExchangeDataCase::kTaifex:
       desc = security.taifex().security_desc();
       break;

src/lib/domain_model/Security.h

@@ -95,27 +95,11 @@ PB_ENUM_TO_JSON(SyntheticContract_PriceCalc);
 
 namespace bts::securities {
 
-/**
- * Gets a descriptor string for a security (should be unique within a particular exchange).
- */
-const std::string &descriptor(const api::Security &sec);
-
 /**
  * Gets a descriptor string for a security (should be unique within a particular exchange).
  */
 const std::string &descriptor(
-    const api::Security &sec, const std::string &default_rv);
-
-/**
- * Gets a descriptor string for a security (should be unique within a particular exchange).
- */
-const std::string &descriptor(const api::Security *sec);
-
-/**
- * Gets a descriptor string for a security (should be unique within a particular exchange).
- */
-const std::string &descriptor(
-    const api::Security *sec, const std::string &default_rv);
+    const api::Security &sec, const std::string &default_rv = "UNDEFINED");
 
 /**
  * Gets a descriptor string for a security that is unique within the entire
@@ -337,11 +321,6 @@ inline Price tick_size(api::Security const &sec) {
    return Price(sec.contract_spec().low_tick());
 }
 
-inline const std::string &descriptor(const api::Security &sec) {
-   static const std::string default_rv = "UNDEFINED";
-   return descriptor(sec, default_rv);
-}
-
 inline const std::string &descriptor(
     const api::Security &sec, const std::string &default_rv) {
    switch (sec.ExchangeData_case()) {
@@ -367,8 +346,6 @@ inline const std::string &descriptor(
       return sec.coinbase().security_desc();
    case api::Security::kOkx:
       return sec.okx().security_desc();
-   case api::Security::kTwse:
-      return sec.twse().security_desc();
    case api::Security::kTaifex:
       return sec.taifex().security_desc();
    default: {
@@ -379,17 +356,6 @@ inline const std::string &descriptor(
    }
    }
 }
-
-inline const std::string &descriptor(const api::Security *sec) {
-   static const std::string default_rv = "UNDEFINED";
-   return sec == nullptr ? default_rv : descriptor(*sec, default_rv);
-}
-
-inline const std::string &descriptor(
-    const api::Security *sec, const std::string &default_rv) {
-   return sec == nullptr ? default_rv : descriptor(*sec, default_rv);
-}
-
 // Looking to round md::IntPrice? Look in IntPrice.h
 
 // computes price_2 - price_1 in ticks

src/lib/domain_model/Security.proto

@@ -21,7 +21,6 @@ enum Venue {
      VENUE_COINBASE = 450;
      VENUE_OKX = 460;
      VENUE_TAIFEX = 500;
-     VENUE_TWSE = 600;
 }
 
 enum SecurityType {
@@ -89,11 +88,6 @@ message CBOEExchangeData {
     int32 class_key = 30;
     string source = 40;
 }
-message TWSEExchangeData {
-    int64 exchange_id = 10;
-    int32 class_key = 30;
-    string security_desc = 20;
-}
 message TAIFEXExchangeData {
     int64 exchange_id = 10;
     int32 class_key = 30;
@@ -266,7 +260,6 @@ message Security {
        CoinbaseExchangeData coinbase = 290;
        OkxExchangeData okx = 300;
        TAIFEXExchangeData taifex = 400;
-       TWSEExchangeData twse = 500;
    }
 }
 message SecurityList {

src/lib/domain_model/SecurityTests.cpp

@@ -1,4 +1,4 @@
-#include <catch2/catch_all.hpp>
+#include "catch/catch.hpp"
 #include <iostream>
 #include <regex>
 

src/lib/domain_model/tests/domain_tests.cpp

@@ -1,5 +1,6 @@
 
 #include "../Security.h"
+#include "catch/catch.hpp"
 #include "lib/core/TimeStamp.h"
 #include "lib/domain_model/Security.h"
 #include "lib/quickbus/rpc/RPCServiceName.h"
@@ -7,7 +8,6 @@
 #include "lib/utils/datetime.h"
 #include <boost/date_time/gregorian/gregorian.hpp>
 #include <boost/date_time/posix_time/posix_time.hpp>
-#include <catch2/catch_all.hpp>
 #include <random>
 
 namespace bts {
@@ -27,7 +27,7 @@ TEST_CASE("TimeStamp", "[TimeStamp]") {
                   day = get_random(1, 28);
          // std::cout << year << "-" << month << "-" << day << std::endl;
          bts::core::TimeStamp ts1(year, month, day);
-         int64_t epoch_ns = ts1.nanos();
+         uint64_t epoch_ns = ts1.nanos();
          bts::core::TimeStamp ts2 = bts::core::TimeStamp{epoch_ns};
          uint32_t year_2, month_2, day_2;
          ts2.to_date(year_2, month_2, day_2);

src/lib/io/JsonReader.cpp

@@ -417,7 +417,7 @@ bool JsonReader::parse_iso_time_stamp_(
 
 bool JsonReader::parse_epoch_nanos_time_stamp_(
     const string &str, TimeStamp &value) const {
-   int64_t nanos = stoll(str);
+   uint64_t nanos = stoull(str);
    value = TimeStamp{nanos};
    return true;
 }

src/lib/io/websocket/tests/WebsocketClientTests.cpp

@@ -23,7 +23,7 @@
 #include <boost/core/addressof.hpp>
 #include <boost/date_time/posix_time/posix_time.hpp>
 #include <boost/date_time/posix_time/posix_time_duration.hpp>
-#include <catch2/catch_all.hpp>
+#include <catch/catch.hpp>
 #include <chrono>
 #include <cstdint>
 #include <cstdlib>

src/lib/io/websocket/websocket_client.cpp

@@ -9,10 +9,8 @@
 #include "lib/io/websocket/websocket_error_code.h"
 #include "lib/io/websocket/websocketpp_exception_formatter.h"
 #include "lib/logging/logging.h"
-#include "lib/utils/LockBox.h"
 #include "lib/utils/bts_assert.h"
 #include "lib/utils/formatters/exception_formatter.h"
-#include "lib/utils/thread_manager.h"
 
 #include <algorithm>
 #include <atomic>
@@ -101,7 +99,6 @@ using namespace std::chrono_literals;
 
 using bts::core::TimeSpan;
 using bts::core::TimeStamp;
-using bts::threading::configure_current_thread;
 using nlohmann::json;
 using std::lock_guard;
 using std::map;
@@ -111,7 +108,6 @@ using std::size_t;
 using std::string;
 using std::thread;
 using std::uint32_t;
-using std::vector;
 using std::chrono::nanoseconds;
 using std::this_thread::sleep_for;
 
@@ -153,8 +149,7 @@ struct WebsocketClient::PImpl {
    optional<CloseHandler> close_handler;
    size_t sub_id_counter{0};
    string server_name;
-   optional<thread> ping_thread;
-   utils::LockBox<vector<thread>> io_threads;
+   optional<thread> io_thread;
    int client_id{client_id_counter++};
    mutex instance_mx;
    State state = State::IDLE;
@@ -201,17 +196,9 @@ struct WebsocketClient::PImpl {
    }
 
    ~PImpl() {
-      close(
-          websocketpp::close::status::going_away,
-          "WebsocketClient::PImpl destructor");
-      io_threads.lock([](auto &ts) {
-         for (auto &t : ts) {
-            if (t.joinable()) {
-               t.join();
-            }
-         }
-         ts.clear();
-      });
+      if (io_thread) {
+         io_thread->join();
+      }
    }
 
    int set_state(State state) {
@@ -322,17 +309,14 @@ struct WebsocketClient::PImpl {
              "io_thread. Websocket io threw websocketpp::exception. {}; "
              "err: {}",
              snapshot(), std::string(ex.what()));
-         bts_assert_msgf(
-             false,
-             "io_thread. Websocket io threw websocketpp::exception. {}; "
+             bts_assert_msgf(false, "io_thread. Websocket io threw websocketpp::exception. {}; "
              "err: {}",
-             snapshot(), std::string(ex.what()));
+               snapshot(), std::string(ex.what()));
       } catch (const std::exception &ex) {
          ERROR_FN(
              "io_thread. Websocket io threw std::exception. {}; err: {}",
              snapshot(), std::string(ex.what()));
-         bts_assert_msgf(
-             false, "io_thread. Websocket io threw std::exception. {}; err: {}",
+         bts_assert_msgf(false, "io_thread. Websocket io threw std::exception. {}; err: {}",
              snapshot(), std::string(ex.what()));
       } catch (...) {
          bts_assert_msgf(
@@ -344,17 +328,6 @@ struct WebsocketClient::PImpl {
       INFO("Websocket 'run' thread end. {}", snapshot());
    }
 
-   void run_in_background(int num_threads) {
-      io_threads.lock([&](vector<thread> &ts) {
-         while (num_threads-- > 0) {
-            ts.emplace_back([&]() {
-               configure_current_thread(fmt::format("{}-io", config.name));
-               run();
-            });
-         }
-      });
-   }
-
    void run_one() {
       try {
          client.run_one();
@@ -377,40 +350,9 @@ struct WebsocketClient::PImpl {
       }
    }
 
-   void ping_continuously(int interval_secs, string payload) {
-      INFO_FN("ping_continuously");
-      if (ping_thread) {
-         // stop the old ping
-         is_continuous_ping = false;
-         ping_thread->join();
-      }
-      is_continuous_ping = true;
-      ping_thread.emplace([&, interval_secs, payload = std::move(payload)]() {
-         configure_current_thread(fmt::format("{}-ping", config.name));
-         const string &url = config.url;
-         DIAG_FN("ping_continuously. Thread starting. url: {}", url);
-         auto last_ping_time = TimeStamp::now();
-         std::this_thread::sleep_for(std::chrono::seconds(interval_secs));
-         while (state == State::OPEN && is_continuous_ping) {
-            const auto now = TimeStamp::now();
-            if ((now - last_ping_time).seconds() > interval_secs) {
-               DIAG_FN("ping_continuously. Pinging websocket. url: {}", url);
-               con->ping(payload);
-               last_ping_time = now;
-            }
-            std::this_thread::sleep_for(std::chrono::seconds(1));
-         }
-         DIAG_FN("ping_continuously. Thread exiting. url: {}", url);
-      });
-   }
-
-   void stop_continuous_ping() { is_continuous_ping = false; }
-
  private:
    static std::atomic<int> client_id_counter;
 
-   bool is_continuous_ping = false;
-
    std::shared_ptr<boost::asio::ssl::context> on_tls_init(
        ConnectionHdl /*hdl*/) {
       INFO_FN("on_tls_init. state: {}; version: {}", state, version);
@@ -537,10 +479,6 @@ void WebsocketClient::connect() { me_->connect(); }
 
 void WebsocketClient::run() { me_->run(); }
 
-void WebsocketClient::run_in_background(int num_threads) {
-   me_->run_in_background(num_threads);
-}
-
 void WebsocketClient::run_one() { me_->run_one(); }
 
 boost::asio::io_context *WebsocketClient::io_context() {
@@ -564,13 +502,6 @@ bool WebsocketClient::is_connected() const {
 
 void WebsocketClient::ping(const string &payload) { me_->con->ping(payload); }
 
-void WebsocketClient::ping_continuously(
-    int interval_secs, const std::string &payload) {
-   return me_->ping_continuously(interval_secs, payload);
-}
-
-void WebsocketClient::stop_continous_ping() { me_->stop_continuous_ping(); }
-
 const string &WebsocketClient::url() const { return me_->config.url; }
 
 } // namespace bts::websocket

src/lib/io/websocket/websocket_client.h

@@ -27,15 +27,12 @@ class WebsocketClient {
    bool is_connected() const;
    boost::asio::io_context *io_context();
    void run();
-   void run_in_background(int num_threads = 1);
    void run_one();
-   OpenHandler &open_handler();
+   CloseHandler &open_handler();
    MessageHandler &message_handler();
    CloseHandler &close_handler();
    websocketpp::lib::error_code send_text(const std::string &text);
    void ping(const std::string &payload = "");
-   void ping_continuously(int interval_secs, const std::string &payload = "");
-   void stop_continous_ping();
 
    const std::string &url() const;
 
@@ -45,3 +42,4 @@ class WebsocketClient {
 };
 
 } // namespace bts::websocket
+

src/lib/jetfix/tests/async_reader_tests.cpp

@@ -1,10 +1,11 @@
+#include "catch/catch.hpp"
+
 #include "lib/jetfix/common/InMsgBuffer.h"
 #include "lib/jetfix/connection/AsyncReader.h"
 #include "lib/jetfix/connection/IConnection.h"
 #include "lib/jetfix/gateway/ICoreGatewayEventSink.h"
 #include "lib/jetfix/tests/sample_fix.h"
 
-#include <catch2/catch_all.hpp>
 #include <fstream>
 
 namespace bts {

src/lib/jetfix/tests/decoding_tests.cpp

@@ -1,3 +1,5 @@
+#include "catch/catch.hpp"
+
 #include "lib/jetfix/codec/FIXFragment.h"
 #include "lib/jetfix/codec/FIXMsg.h"
 #include "lib/jetfix/codec/FIXParser.h"
@@ -8,8 +10,6 @@
 #include "lib/jetfix/connection/IConnectionObserver.h"
 #include "lib/utils/TimeStamp.h"
 
-#include <catch2/catch_all.hpp>
-
 namespace bts {
 namespace jetfix {
 

src/lib/jetfix/tests/fix_encoding_stream_tests.cpp

@@ -1,6 +1,6 @@
-#include "lib/jetfix/codec/FIXEncodingStream.h"
+#include "catch/catch.hpp"
 
-#include <catch2/catch_all.hpp>
+#include "lib/jetfix/codec/FIXEncodingStream.h"
 #include <fstream>
 
 namespace bts {

src/lib/jetfix/tests/fix_fragment_tests.cpp

@@ -1,9 +1,9 @@
+#include "catch/catch.hpp"
+
 #include "lib/jetfix/codec/FIXFragment.h"
 #include "lib/jetfix/codec/Fields.h"
 #include "lib/utils/datetime.h"
 
-#include <catch2/catch_all.hpp>
-
 namespace bts {
 namespace jetfix {
 

src/lib/jetfix/tests/jetfix_performance_tests.cpp

@@ -1,3 +1,5 @@
+#include "catch/catch.hpp"
+
 #include "lib/jetfix/codec/FIXFragment.h"
 #include "lib/jetfix/codec/FIXMsg.h"
 #include "lib/jetfix/codec/FIXParser.h"
@@ -9,8 +11,6 @@
 #include "lib/logging/logging.h"
 #include "lib/utils/TimeStamp.h"
 
-#include <catch2/catch_all.hpp>
-
 namespace bts {
 namespace jetfix {
 

src/lib/jetfix/tests/store_tests.cpp

@@ -1,3 +1,5 @@
+#include "catch/catch.hpp"
+
 #include "lib/jetfix/codec/FIXEncodingStream.h"
 #include "lib/jetfix/common/InMsgBuffer.h"
 #include "lib/jetfix/common/OutFIX.h"
@@ -12,7 +14,6 @@
 #include <boost/date_time.hpp>
 #include <boost/date_time/gregorian/gregorian.hpp>
 #include <boost/filesystem.hpp>
-#include <catch2/catch_all.hpp>
 
 namespace bts {
 namespace jetfix {

src/lib/logging/ccapi_logger.h

@@ -1,8 +1,8 @@
 #include "logging.h"
 
 #include "LoggerHub.h"
+#include "lib/core/LockGuard.h"
 #include "lib/utils/bts_assert.h"
-#include "lib/utils/bts_lock_guard.h"
 
 #include <boost/circular_buffer.hpp>
 #include <boost/container/flat_set.hpp>
@@ -79,7 +79,7 @@ class CcapiLogger final : public Logger {
    // size would be limited, but not without cost in additional overhead for
    // lookups.
    const std::string &store_file_name_(const std::string &file_name) {
-      BTS_LOCK_GUARD(instance_mx_);
+      LOCK_GUARD(instance_mx_);
       return *file_name_store_.insert(file_name).first;
    }
 };

src/lib/logging/tests/logging_tests.cpp

@@ -1,9 +1,9 @@
+#include "catch/catch.hpp"
 #include "lib/config/config.h"
 #include "lib/config/protos/logging.pb.h"
 #include "lib/logging/logging.h"
 #include "lib/utils/bts_chrono.h"
 
-#include <catch2/catch_all.hpp>
 #include <fmt/format.h>
 #include <fmt/chrono.h>
 

src/lib/logging/trace_log.h

@@ -1,11 +0,0 @@
-#pragma once
-
-#define LOG_TRACE(subsystem, level, scope, name)                                     \
-   LOG(subsystem, level,                                                             \
-       R"([trace.json]{{ "scope": "{}", "name": "{}", "params": null, "ts": {} }})", \
-       scope, name, static_cast<std::int64_t>(bts::log::log_ts()))
-
-#define LOG_TRACE_PARAMS(subsystem, level, scope, name, params)                    \
-   LOG(subsystem, level,                                                           \
-       R"([trace.json]{{ "scope": "{}", "name": "{}", "params": {}, "ts": {} }})", \
-       scope, name, params, static_cast<std::int64_t>(bts::log::log_ts()))

src/lib/md/GeneralMarketDataListener.cpp

@@ -19,8 +19,6 @@
 #include "lib/domain/SpreadSecurity.h"
 #include "lib/domain/TradeRecord.h"
 #include "lib/domain_model/Security.pb.h"
-#include "lib/domain_model/common.h"
-#include "lib/io/JsonDumper.h"
 #include "lib/logging/logging.h"
 #include "lib/md/GeneralSecurityIndex.h"
 #include "lib/md/IMarketDataListener.h"
@@ -131,16 +129,6 @@ struct IncrementalBookData {
             int insert_position = 0;
             for (size_t i = 0; i < book_side.size();
                  i++) { //not sure how to access book
-               if (book_side[i].price == op.price) {
-                  IMarketDataListener::BookOp new_op;
-                  new_op.id = op.id;
-                  new_op.op_type = IMarketDataListener::BookOpType::MODIFY;
-                  new_op.price = op.price;
-                  new_op.quantity = op.quantity;
-                  new_op.side = op.side;
-
-                  return update_incremental(new_op);
-               }
                if (book_side[i].price != 0 && book_side[i].price > op.price) {
                   //book_side is the book we have ?
                   //op.price is what we received ?
@@ -199,25 +187,16 @@ struct IncrementalBookData {
          switch (op.op_type) {
          case IMarketDataListener::BookOpType::Add: {
             int insert_position = 0;
-            for (int i = 0; i < fmin(book_side.size(),5); i++) {
-               if (book_side[i].price == op.price) {
-                  IMarketDataListener::BookOp new_op;
-                  new_op.id = op.id;
-                  new_op.op_type = IMarketDataListener::BookOpType::MODIFY;
-                  new_op.price = op.price;
-                  new_op.quantity = op.quantity;
-                  new_op.side = op.side;
-
-                  return update_incremental(new_op);
+            for (int i = 0; i < fmin(book_side.size(), 5); i++) {
+               if (book_side[i].price != 0 && book_side[i].price < op.price) {
+                  insert_position++;
+                  continue;
                }
-                if (book_side[i].price != 0 && book_side[i].price < op.price) {
-                    insert_position++;
-                    continue;
-                }
-            }//need i to be at the last level
-            for (int i = fmin(book_side.size()-1,5-1); i > insert_position; i--) {
-                book_side[i].price = book_side[i - 1].price;
-                book_side[i].total_quantity = book_side[i - 1].total_quantity;
+            } //need i to be at the last level
+            for (int i = fmin(book_side.size() - 1, 5 - 1); i > insert_position;
+                 i--) {
+               book_side[i].price = book_side[i - 1].price;
+               book_side[i].total_quantity = book_side[i - 1].total_quantity;
             }
             book_side[insert_position].price = op.price;
             book_side[insert_position].total_quantity = op.quantity;
@@ -504,157 +483,21 @@ void GeneralMarketDataListener::on_book_increment(
    bts_assert(book_ptr != nullptr);
    _on_book(*book_ptr);
 }
-co2::AggressorSide try_fill_agg_side(
-    const domain::Price &trade_price,
-    const co2::AggressorSide &current_side,
-    const domain::Book *book,
-    const domain::SecurityID &id) {
-   if (current_side != co2::AGGRESSOR_SIDE_NONE)
-      return current_side;
-   if (!book) {
-      LOG(md, Info,
-          "No book found for sec_id: {}. Processing trade with aggressor "
-          "side set to NONE.",
-          id);
-      return current_side;
-   }
-   // Access ask-side and bid-side levels
-   auto book_ask_side = book->asks();
-   auto book_bid_side = book->bids();
-   if (book_ask_side.is_populated() &&
-       trade_price >=
-           book_ask_side[0]
-               .price) //this means the trade happened on the ask side
-   {
-      LOG(md, Info, "Trade Price: {}, Book_Ask[0] Price: {}", trade_price,
-          book_ask_side[0].price);
-      return co2::AGGRESSOR_SIDE_BUY;
-   }
-   if (book_bid_side.is_populated() &&
-       trade_price <=
-           book_bid_side[0]
-               .price) //this means the trade happened on the bid side
-   {
-      LOG(md, Info, "Trade Price: {}, Book_Ask[0] Price: {}", trade_price,
-          book_bid_side[0].price);
 
-      return co2::AGGRESSOR_SIDE_SELL;
-   }
-   std::string book_str =
-       "Bids: " + bts::io::JsonDumper::dump(book_bid_side, 2) +
-       "\nAsks: " + bts::io::JsonDumper::dump(book_ask_side, 2);
-   LOG(md, Info,
-       "Could not determine Aggressor, Trade Price: {}, Current Book: {}",
-       trade_price, book_str);
-
-   return co2::AGGRESSOR_SIDE_NONE;
-}
 void GeneralMarketDataListener::on_trades(
     const std::vector<domain::TradeRecord const *> &trades) {
    for (auto tp : trades) {
-
-      std::ostringstream log_stream;
-
-      // Log the important fields from the TradeRecord pointer
-      log_stream << "TradePointer - sec_id: " << tp->sec_id
-                 << ", agg_side: " << tp->agg_side
-                 << ", trade_price: " << tp->trade_price
-                 << ", trade_quantity: " << tp->trade_quantity.value()
-                 << ", time_stamp: " << tp->time_stamp.millis();
-
-      // Log the trade data
-      LOG(md, Diag, "logging tp {}", log_stream.str());
-
-      auto handle =
-          _find_handle(tp->sec_id); // might be the reason behind missing trades
-      auto aggressor_side = convert_agg_side(tp->agg_side);
-
-      // Log the initial aggressor_side
-      LOG(md, Diag, "Initial aggressor side for sec_id: {} is: {}", tp->sec_id,
-          static_cast<int>(aggressor_side));
-
-      if (handle == nullptr) {
-         LOG(md, Info, "No handle found for sec_id: {}", tp->sec_id);
-         continue;
-      }
-      aggressor_side = try_fill_agg_side(
-          tp->trade_price, aggressor_side,
-          me->inc_book_data.get_const_book(tp->sec_id), tp->sec_id);
-
+      auto handle = _find_handle(tp->sec_id);
       auto trade_ts = datetime::ll_from_unix_epoch_ms(tp->time_stamp.millis());
-      // Process the trade even if aggressor_side is NONE
       if (handle != nullptr) {
          handle->handle.add_trade(
              utils::TimeStamp::now(),
              PriceSize(
                  convert_price(tp->trade_price), tp->trade_quantity.value()),
-             trade_ts, aggressor_side, 0, false);
-         LOG(md, Diag, "Trade processed for sec_id: {}", tp->sec_id);
+             trade_ts, convert_agg_side(tp->agg_side), 0, false);
       }
    }
 }
-//Needed for diag for trade gaps 
-// void GeneralMarketDataListener::on_trades(
-//     const std::vector<domain::TradeRecord const *> &trades) {
-//    for (auto tp : trades) {
-
-//          for (auto tp : trades) {
-//     std::ostringstream log_stream;
-
-//     // Log the important fields from the TradeRecord pointer
-//     log_stream << "TradePointer - sec_id: " << tp->sec_id
-//                << ", agg_side: " << tp->agg_side
-//                << ", trade_price: " << tp->trade_price
-//                << ", trade_quantity: " << tp->trade_quantity.value()
-//                << ", time_stamp: " << tp->time_stamp.millis();
-
-//     // Log the trade data
-//     LOG(md, Info, "logging tp {}",log_stream.str());}
-
-//       auto handle =
-//           _find_handle(tp->sec_id); //might be the reason behind missing trades
-//       auto aggressor_side = convert_agg_side(tp->agg_side);
-//       if (aggressor_side == co2::AggressorSide::AGGRESSOR_SIDE_NONE) {
-//          auto book = me->inc_book_data.get_const_book(tp->sec_id);
-//          // Check if the book is empty
-//          if (!book) {
-//             LOG(md, Info, "book pointer log");
-//             LOG(md, Info, "Processing trade with sec_id: {}", tp->sec_id);
-//             aggressor_side =
-//                 co2::AggressorSide::AGGRESSOR_SIDE_NONE; // Reset aggressor side
-//             continue; // Skip processing this trade
-//          }else {
-//     LOG(md, Info, "Book found for sec_id: {} with size: {}", tp->sec_id, book->size());
-// }
-
-//          // Access ask-side and bid-side levels
-//          auto book_ask_side = book->asks();
-//          auto book_bid_side = book->bids();
-//           if (!book_ask_side.is_populated() || !book_bid_side.is_populated()) {
-//               aggressor_side = co2::AggressorSide::AGGRESSOR_SIDE_NONE; // Reset
-//               continue; // Skip processing this trade
-//           }
-//            // Determine aggressor side based on trade price
-//           if (tp->trade_price <= book_ask_side[0].price) {
-//               aggressor_side = co2::AggressorSide::AGGRESSOR_SIDE_BUY;
-//           } else if (tp->trade_price >= book_bid_side[0].price) {
-//               aggressor_side = co2::AggressorSide::AGGRESSOR_SIDE_SELL;
-//          }
-//       }
-
-//       //     // Check if ask and bid sides are populated
-
-//       // }
-//       auto trade_ts = datetime::ll_from_unix_epoch_ms(tp->time_stamp.millis());
-//       if (handle != nullptr) {
-//          handle->handle.add_trade(
-//              utils::TimeStamp::now(),
-//              PriceSize(
-//                  convert_price(tp->trade_price), tp->trade_quantity.value()),
-//              trade_ts, aggressor_side, 0, false);
-//       }
-//    }
-// }
 
 void GeneralMarketDataListener::on_rfqs(
     const std::vector<domain::RFQ const *> &) {}

src/lib/md/Shm/FixedSecurity.cpp

@@ -222,10 +222,6 @@ void convert_security(const FixedSecurity &source, Security *presult) {
       dest.mutable_okx()->set_security_desc(source.security_desc());
       dest.mutable_okx()->set_exchange_id(source.numeric_id);
       break;
-   case Venue::VENUE_TWSE:
-      dest.mutable_twse()->set_security_desc(source.security_desc());
-      dest.mutable_twse()->set_exchange_id(source.numeric_id);
-      break;
    case Venue::VENUE_TAIFEX:
       dest.mutable_taifex()->set_security_desc(source.security_desc());
       dest.mutable_taifex()->set_exchange_id(source.numeric_id);
@@ -338,8 +334,6 @@ uint64_t exchange_id(const Security &security) {
       return security.binance_usds_futures().exchange_id();
    case Security::ExchangeDataCase::kOkx:
       return security.okx().exchange_id();
-   case Security::ExchangeDataCase::kTwse:
-      return security.twse().exchange_id();
    case Security::ExchangeDataCase::kTaifex:
       return security.taifex().exchange_id();
    // Debatable whether this should be an error or not.
@@ -373,15 +367,10 @@ void convert_security(
       dest.security_desc(src.cboe().security_desc());
       dest.cboe_class_key = src.cboe().class_key();
       break;
-   case Security::ExchangeDataCase::kTwse:
-      dest.numeric_id = src.twse().exchange_id();
-      dest.security_desc(src.twse().security_desc());
-      // dest.cboe_class_key = src.twse().class_key();
-      break;
    case Security::ExchangeDataCase::kTaifex:
-      dest.numeric_id = src.taifex().exchange_id();
-      dest.security_desc(src.taifex().security_desc());
-      // dest.cboe_class_key = src.taifex().class_key();
+      dest.numeric_id = src.deribit().exchange_id();
+      dest.security_desc(src.deribit().security_desc());
+      dest.cboe_class_key = src.taifex().class_key();
       break;
    case Security::ExchangeDataCase::kCfe:
       dest.cboe_class_key = src.cfe().class_key();