fix build: Trivial changes to fix emumba_deribit build

* Use "Unix Makefiles" generator when building websocket-library. - Ninja tries to build things in parallel, and doesn't build the ExternalProject dependencies in the right order. Consequently, the websocket-library build fails because it can't find "libspdlog.a". * Replace a few leftover SPDLOG_ERROR macros w/ LOG_ERROR (the latter logs in our system, not spdlog).
scafolding
2025-01-10 19:36:18 +00:00 · 2025-01-09 19:56:54 +00:00 · 2025-01-06 05:00:04 +00:00 · 2025-01-06 03:52:25 +00:00 · 2025-01-05 15:58:29 -05:00 · 2025-01-05 15:29:11 -05:00
301 changed files with 21608 additions and 17776 deletions
@@ -9,7 +9,7 @@
      ]
    },
    "fable": {
-      "version": "4.0.6",
+      "version": "4.24.0",
      "commands": [
        "fable"
      ]
@@ -27,4 +27,4 @@
      ]
    }
  }
-}
+}
@@ -31,7 +31,24 @@
      "type": "cppdbg",
      "request": "launch",
      "program": "/home/bts/spark/build/sparkd-debug-clang/server/sparkd",
-      "args": ["run"],
+      "args": [
+        "run"
+      ],
+      "stopAtEntry": false,
+      "cwd": "/home/bts/spark/build/sparkd-debug-clang/server",
+      "environment": [],
+      "externalConsole": false,
+      "MIMode": "gdb",
+    },
+    {
+      "name": "spreader tests2",
+      "type": "cppdbg",
+      "request": "launch",
+      "program": "/home/bts/spark/build/sparkd-debug-clang/server/sparkd",
+      "args": [
+        "test",
+        "[spreader2]"
+      ],
      "stopAtEntry": false,
      "cwd": "/home/bts/spark/build/sparkd-debug-clang/server",
      "environment": [],
@@ -110,20 +127,104 @@
    },
    {
      "name": "Sparkd",
-      "type": "coreclr",
+      "type": "cppdbg",
      "request": "launch",
-      "preLaunchTask": "Build Sparkd (Debug | Clang)",
      "program": "",
      "linux": {
-        "program": "sparkd",
-      },
-      "args": [
-        "run"
-      ],
-      "cwd": "build/sparkd-debug-clang/server",
-      "stopAtEntry": false,
-      "console": "integratedTerminal"
-    }
+        "preLaunchTask": "Build Sparkd (Debug)",
+        "program": "${workspaceFolder}/build/sparkd-debug-clang/server/sparkd",
+        "args": [
+          "run"
+        ],
+        "stopAtEntry": false,
+        "cwd": "${workspaceFolder}/build/sparkd-debug-clang/server",
+        "environment": [],
+        "externalConsole": false,
+        "MIMode": "gdb",
+        "setupCommands": [
+          {
+            "description": "Enable pretty-printing for gdb",
+            "text": "-enable-pretty-printing",
+            "ignoreFailures": true
+          }
+        ]
+      }
+    },
+    {
+      "name": "Sparkd (Skip Build)",
+      "type": "cppdbg",
+      "request": "launch",
+      "program": "",
+      "linux": {
+        "program": "${workspaceFolder}/build/sparkd-debug-clang/server/sparkd",
+        "args": [
+          "run"
+        ],
+        "stopAtEntry": false,
+        "cwd": "${workspaceFolder}/build/sparkd-debug-clang/server",
+        "environment": [],
+        "externalConsole": false,
+        "MIMode": "gdb",
+        "setupCommands": [
+          {
+            "description": "Enable pretty-printing for gdb",
+            "text": "-enable-pretty-printing",
+            "ignoreFailures": true
+          }
+        ]
+      }
+    },
+    {
+      "name": "Sparkd Test",
+      "type": "cppdbg",
+      "request": "launch",
+      "program": "",
+      "linux": {
+        "preLaunchTask": "Build Sparkd (Debug)",
+        "program": "${workspaceFolder}/build/sparkd-debug-clang/server/sparkd",
+        "args": [
+          "test",
+          "${input:sparkdTest}",
+        ],
+        "stopAtEntry": false,
+        "cwd": "${workspaceFolder}/build/sparkd-debug-clang/server",
+        "environment": [],
+        "externalConsole": false,
+        "MIMode": "gdb",
+        "setupCommands": [
+          {
+            "description": "Enable pretty-printing for gdb",
+            "text": "-enable-pretty-printing",
+            "ignoreFailures": true
+          }
+        ]
+      }
+    },
+    {
+      "name": "Sparkd Test (Skip Build)",
+      "type": "cppdbg",
+      "request": "launch",
+      "program": "",
+      "linux": {
+        "program": "${workspaceFolder}/build/sparkd-debug-clang/server/sparkd",
+        "args": [
+          "test",
+          "${input:sparkdTest}",
+        ],
+        "stopAtEntry": false,
+        "cwd": "${workspaceFolder}/build/sparkd-debug-clang/server",
+        "environment": [],
+        "externalConsole": false,
+        "MIMode": "gdb",
+        "setupCommands": [
+          {
+            "description": "Enable pretty-printing for gdb",
+            "text": "-enable-pretty-printing",
+            "ignoreFailures": true
+          }
+        ]
+      }
+    },
  ],
  "inputs": [
    {
@@ -154,5 +255,10 @@
      "description": "Config Server URL",
      "default": "http://localhost:8000/config",
    },
+    {
+      "id": "sparkdTest",
+      "type": "promptString",
+      "description": "Test to run",
+    },
  ]
-}
+}
@@ -1,5 +1,6 @@
 cmake_minimum_required(VERSION 3.12)
 project(spark)
+include(GNUInstallDirs)
 set(CMAKE_CXX_STANDARD 20)

 if(WIN32)
@@ -49,8 +50,6 @@ else()
  # make sparkd look for libspark.so in its directory
  set(CMAKE_BUILD_WITH_INSTALL_RPATH TRUE)
  set(CMAKE_INSTALL_RPATH ".")
-  set(CMAKE_INSTALL_PREFIX "${CMAKE_SOURCE_DIR}/install")
-  set(SPARK_INSTALL_PREFIX "${CMAKE_INSTALL_PREFIX}/$<CONFIG>")

  if(BUILD_SPARKD)
    # Note: All macros that begin w/ 'CCAPI' are ccapi's macros (defined and/or
@@ -89,9 +88,15 @@ else()
    add_compile_definitions(BINANCE_US_PRICE_STATS_PATH="/api/v3/ticker/24hr")

    # ** Binance USDS Futures **
-    add_compile_definitions(CCAPI_BINANCE_USDS_FUTURES_URL_REST_BASE="https://dapi.binance.com")
-    add_compile_definitions(BINANCE_USDS_FUTURES_INSTRUMENTS_PATH="/dapi/v1/exchangeInfo")
-    add_compile_definitions(BINANCE_USDS_FUTURES_PRICE_STATS_PATH="/dapi/v1/ticker/24hr")
+    add_compile_definitions(CCAPI_BINANCE_USDS_FUTURES_URL_REST_BASE="https://fapi.binance.com")
+    add_compile_definitions(BINANCE_USDS_FUTURES_INSTRUMENTS_PATH="/fapi/v1/exchangeInfo")
+    add_compile_definitions(BINANCE_USDS_FUTURES_PRICE_STATS_PATH="/fapi/v1/ticker/24hr")
+    add_compile_definitions(BINANCE_USDS_FUTURES_USER_DATA_PATH="/api/v3/userDataStream")
+
+    # ** Binance Coin Futures **
+    add_compile_definitions(CCAPI_BINANCE_COIN_FUTURES_URL_REST_BASE="https://dapi.binance.com")
+    add_compile_definitions(BINANCE_COIN_FUTURES_INSTRUMENTS_PATH="/dapi/v1/exchangeInfo")
+    add_compile_definitions(BINANCE_COIN_FUTURES_PRICE_STATS_PATH="/dapi/v1/ticker/24hr")

    # ** Deribit **
    add_compile_definitions(DERIBIT_INSTRUMENT_PATH="/api/v2/public/get_instrument")
@@ -111,6 +116,8 @@ else()

  # subdirs
  if (BUILD_SPARKD)
+    add_subdirectory(src/lib_binance_execution)
+    add_subdirectory(src/lib_deribit_emumba)
    add_subdirectory(src/server/sparkd)
    add_subdirectory(src/data_files)
  endif()
@@ -147,11 +154,14 @@ else()

    install(
      DIRECTORY "${CMAKE_BINARY_DIR}/server/plugins"
-      DESTINATION "${SPARK_INSTALL_PREFIX}/bin")
+      DESTINATION "${CMAKE_INSTALL_BINDIR}")
    install(
      FILES
      "${SPARK_LIB}/libACE-5.7.5.so"
      "${SPARK_LIB}/libOnixS.IceImpactMulticastPriceFeedHandlerCpp.so.3.16.0.0"
-      DESTINATION "${SPARK_INSTALL_PREFIX}/bin")
+      "${SPARK_LIB}/libonixs-fix-engine.so.4.10.1"
+      "${SPARK_LIB}/libonixs-fix-engine.so.4"
+      "${SPARK_LIB}/libonixs-fix-engine.so"
+      DESTINATION "${CMAKE_INSTALL_BINDIR}")
  endif()
 endif()
@@ -61,7 +61,7 @@
        },
        "CMAKE_CXX_STANDARD": "20",
        "CMAKE_INSTALL_RPATH": ".",
-        "CMAKE_INSTALL_PREFIX": "${sourceDir}/install/${presetName}/sparkd",
+        "CMAKE_INSTALL_PREFIX": "${sourceDir}/install/${presetName}",
        "CMAKE_EXE_LINKER_FLAGS": "-Wno-undef",
        "EXTERNALS_ROOT": "${sourceDir}/externals/linux",
        "BUILD_SDK": {
@@ -302,16 +302,7 @@
        "lhs": "${hostSystemName}",
        "rhs": "Linux"
      },
-      "hidden": true,
-      "targets": [
-        "data_files_dir",
-        "admin_web_app_bundle",
-        "install",
-        "libspark",
-        "rapidjson_project",
-        "server_layout",
-        "sparkd"
-      ]
+      "hidden": true
    },

    {
@@ -14,6 +14,34 @@ Project("{FAE04EC0-301F-11D3-BF4B-00C04F79EFBC}") = "spark_test", "src\client\sp
 EndProject
 Project("{FAE04EC0-301F-11D3-BF4B-00C04F79EFBC}") = "Spark.Console", "src\client\Spark.Console\Spark.Console.csproj", "{89D666E6-B500-498F-B026-4D167F68ADFC}"
 EndProject
+Project("{2150E333-8FDC-42A3-9474-1A3956D46DE8}") = "src", "src", "{267F81EF-A8AB-49FD-9F56-D7EC3C1F7C7C}"
+EndProject
+Project("{2150E333-8FDC-42A3-9474-1A3956D46DE8}") = "DataStreamSuite", "DataStreamSuite", "{C09ED521-D2DF-40BC-A76F-EB3CB870B2F6}"
+EndProject
+Project("{FAE04EC0-301F-11D3-BF4B-00C04F79EFBC}") = "DataStreamCore", "src\DataStreamSuite\DataStreamCore\DataStreamCore.csproj", "{EF26A475-6101-4C9C-A717-D60BB92A2302}"
+EndProject
+Project("{FAE04EC0-301F-11D3-BF4B-00C04F79EFBC}") = "SharedStreams", "src\DataStreamSuite\SharedStreams\SharedStreams.csproj", "{911B08A3-515A-441D-84C8-8A43740B5FC1}"
+EndProject
+Project("{FAE04EC0-301F-11D3-BF4B-00C04F79EFBC}") = "PubSub", "src\DataStreamSuite\PubSub\PubSub.csproj", "{14140310-2233-430A-821B-F3058A25234B}"
+EndProject
+Project("{FAE04EC0-301F-11D3-BF4B-00C04F79EFBC}") = "BaseUtilities", "src\DataStreamSuite\BaseUtilities\BaseUtilities.csproj", "{43146EA8-6FCC-46B8-88B0-42F9BB5E6E0A}"
+EndProject
+Project("{FAE04EC0-301F-11D3-BF4B-00C04F79EFBC}") = "ExpressionCore", "src\DataStreamSuite\ExpressionCore\ExpressionCore.csproj", "{6F22DB96-6EAB-4034-99D9-287021C36F01}"
+EndProject
+Project("{FAE04EC0-301F-11D3-BF4B-00C04F79EFBC}") = "ArithmeticExpressions", "src\DataStreamSuite\ArithmeticExpressions\ArithmeticExpressions.csproj", "{A5C33956-A41F-49B3-A41D-1082CBA59D02}"
+EndProject
+Project("{FAE04EC0-301F-11D3-BF4B-00C04F79EFBC}") = "DataStreamExpressions", "src\DataStreamSuite\DataStreamExpressions\DataStreamExpressions.csproj", "{2FE745C3-1EDF-4421-8CD7-A63EE0C1AE43}"
+EndProject
+Project("{FAE04EC0-301F-11D3-BF4B-00C04F79EFBC}") = "Bts.ColumnSortExpressions", "src\DataStreamSuite\Bts.ColumnSortExpressions\Bts.ColumnSortExpressions.csproj", "{FDA791E0-2B91-4AD5-B91D-CF920A4C4E38}"
+EndProject
+Project("{FAE04EC0-301F-11D3-BF4B-00C04F79EFBC}") = "ColumnMapExpressions", "src\DataStreamSuite\ColumnMapExpressions\ColumnMapExpressions.csproj", "{B945A183-2174-4ACB-B289-8A5C546656FE}"
+EndProject
+Project("{FAE04EC0-301F-11D3-BF4B-00C04F79EFBC}") = "BooleanExpressions", "src\DataStreamSuite\BooleanExpressions\BooleanExpressions.csproj", "{9F6D7DE8-185B-4C1D-B809-3314D092DF28}"
+EndProject
+Project("{FAE04EC0-301F-11D3-BF4B-00C04F79EFBC}") = "Bts.ColumnExpressions", "src\DataStreamSuite\Bts.ColumnExpressions\Bts.ColumnExpressions.csproj", "{6BD3F6D8-B6FC-44E8-B8FE-06352F638B9B}"
+EndProject
+Project("{FAE04EC0-301F-11D3-BF4B-00C04F79EFBC}") = "DataStream", "src\DataStreamSuite\DataStream\DataStream.csproj", "{39AEA075-6457-46C0-8821-1158BE6EFA57}"
+EndProject
 Global
 	GlobalSection(SolutionConfigurationPlatforms) = preSolution
 		Debug|x64 = Debug|x64
@@ -70,6 +98,102 @@ Global
 		{89D666E6-B500-498F-B026-4D167F68ADFC}.Release|x64.Build.0 = Release|x64
 		{89D666E6-B500-498F-B026-4D167F68ADFC}.RelWithDebInfo|x64.ActiveCfg = Debug|x64
 		{89D666E6-B500-498F-B026-4D167F68ADFC}.RelWithDebInfo|x64.Build.0 = Debug|x64
+		{EF26A475-6101-4C9C-A717-D60BB92A2302}.Debug|x64.ActiveCfg = Debug|Any CPU
+		{EF26A475-6101-4C9C-A717-D60BB92A2302}.Debug|x64.Build.0 = Debug|Any CPU
+		{EF26A475-6101-4C9C-A717-D60BB92A2302}.MinSizeRel|x64.ActiveCfg = Debug|Any CPU
+		{EF26A475-6101-4C9C-A717-D60BB92A2302}.MinSizeRel|x64.Build.0 = Debug|Any CPU
+		{EF26A475-6101-4C9C-A717-D60BB92A2302}.Release|x64.ActiveCfg = Release|Any CPU
+		{EF26A475-6101-4C9C-A717-D60BB92A2302}.Release|x64.Build.0 = Release|Any CPU
+		{EF26A475-6101-4C9C-A717-D60BB92A2302}.RelWithDebInfo|x64.ActiveCfg = Debug|Any CPU
+		{EF26A475-6101-4C9C-A717-D60BB92A2302}.RelWithDebInfo|x64.Build.0 = Debug|Any CPU
+		{911B08A3-515A-441D-84C8-8A43740B5FC1}.Debug|x64.ActiveCfg = Debug|Any CPU
+		{911B08A3-515A-441D-84C8-8A43740B5FC1}.Debug|x64.Build.0 = Debug|Any CPU
+		{911B08A3-515A-441D-84C8-8A43740B5FC1}.MinSizeRel|x64.ActiveCfg = Debug|Any CPU
+		{911B08A3-515A-441D-84C8-8A43740B5FC1}.MinSizeRel|x64.Build.0 = Debug|Any CPU
+		{911B08A3-515A-441D-84C8-8A43740B5FC1}.Release|x64.ActiveCfg = Release|Any CPU
+		{911B08A3-515A-441D-84C8-8A43740B5FC1}.Release|x64.Build.0 = Release|Any CPU
+		{911B08A3-515A-441D-84C8-8A43740B5FC1}.RelWithDebInfo|x64.ActiveCfg = Debug|Any CPU
+		{911B08A3-515A-441D-84C8-8A43740B5FC1}.RelWithDebInfo|x64.Build.0 = Debug|Any CPU
+		{14140310-2233-430A-821B-F3058A25234B}.Debug|x64.ActiveCfg = Debug|Any CPU
+		{14140310-2233-430A-821B-F3058A25234B}.Debug|x64.Build.0 = Debug|Any CPU
+		{14140310-2233-430A-821B-F3058A25234B}.MinSizeRel|x64.ActiveCfg = Debug|Any CPU
+		{14140310-2233-430A-821B-F3058A25234B}.MinSizeRel|x64.Build.0 = Debug|Any CPU
+		{14140310-2233-430A-821B-F3058A25234B}.Release|x64.ActiveCfg = Release|Any CPU
+		{14140310-2233-430A-821B-F3058A25234B}.Release|x64.Build.0 = Release|Any CPU
+		{14140310-2233-430A-821B-F3058A25234B}.RelWithDebInfo|x64.ActiveCfg = Debug|Any CPU
+		{14140310-2233-430A-821B-F3058A25234B}.RelWithDebInfo|x64.Build.0 = Debug|Any CPU
+		{43146EA8-6FCC-46B8-88B0-42F9BB5E6E0A}.Debug|x64.ActiveCfg = Debug|Any CPU
+		{43146EA8-6FCC-46B8-88B0-42F9BB5E6E0A}.Debug|x64.Build.0 = Debug|Any CPU
+		{43146EA8-6FCC-46B8-88B0-42F9BB5E6E0A}.MinSizeRel|x64.ActiveCfg = Debug|Any CPU
+		{43146EA8-6FCC-46B8-88B0-42F9BB5E6E0A}.MinSizeRel|x64.Build.0 = Debug|Any CPU
+		{43146EA8-6FCC-46B8-88B0-42F9BB5E6E0A}.Release|x64.ActiveCfg = Release|Any CPU
+		{43146EA8-6FCC-46B8-88B0-42F9BB5E6E0A}.Release|x64.Build.0 = Release|Any CPU
+		{43146EA8-6FCC-46B8-88B0-42F9BB5E6E0A}.RelWithDebInfo|x64.ActiveCfg = Debug|Any CPU
+		{43146EA8-6FCC-46B8-88B0-42F9BB5E6E0A}.RelWithDebInfo|x64.Build.0 = Debug|Any CPU
+		{6F22DB96-6EAB-4034-99D9-287021C36F01}.Debug|x64.ActiveCfg = Debug|Any CPU
+		{6F22DB96-6EAB-4034-99D9-287021C36F01}.Debug|x64.Build.0 = Debug|Any CPU
+		{6F22DB96-6EAB-4034-99D9-287021C36F01}.MinSizeRel|x64.ActiveCfg = Debug|Any CPU
+		{6F22DB96-6EAB-4034-99D9-287021C36F01}.MinSizeRel|x64.Build.0 = Debug|Any CPU
+		{6F22DB96-6EAB-4034-99D9-287021C36F01}.Release|x64.ActiveCfg = Release|Any CPU
+		{6F22DB96-6EAB-4034-99D9-287021C36F01}.Release|x64.Build.0 = Release|Any CPU
+		{6F22DB96-6EAB-4034-99D9-287021C36F01}.RelWithDebInfo|x64.ActiveCfg = Debug|Any CPU
+		{6F22DB96-6EAB-4034-99D9-287021C36F01}.RelWithDebInfo|x64.Build.0 = Debug|Any CPU
+		{A5C33956-A41F-49B3-A41D-1082CBA59D02}.Debug|x64.ActiveCfg = Debug|Any CPU
+		{A5C33956-A41F-49B3-A41D-1082CBA59D02}.Debug|x64.Build.0 = Debug|Any CPU
+		{A5C33956-A41F-49B3-A41D-1082CBA59D02}.MinSizeRel|x64.ActiveCfg = Debug|Any CPU
+		{A5C33956-A41F-49B3-A41D-1082CBA59D02}.MinSizeRel|x64.Build.0 = Debug|Any CPU
+		{A5C33956-A41F-49B3-A41D-1082CBA59D02}.Release|x64.ActiveCfg = Release|Any CPU
+		{A5C33956-A41F-49B3-A41D-1082CBA59D02}.Release|x64.Build.0 = Release|Any CPU
+		{A5C33956-A41F-49B3-A41D-1082CBA59D02}.RelWithDebInfo|x64.ActiveCfg = Debug|Any CPU
+		{A5C33956-A41F-49B3-A41D-1082CBA59D02}.RelWithDebInfo|x64.Build.0 = Debug|Any CPU
+		{2FE745C3-1EDF-4421-8CD7-A63EE0C1AE43}.Debug|x64.ActiveCfg = Debug|Any CPU
+		{2FE745C3-1EDF-4421-8CD7-A63EE0C1AE43}.Debug|x64.Build.0 = Debug|Any CPU
+		{2FE745C3-1EDF-4421-8CD7-A63EE0C1AE43}.MinSizeRel|x64.ActiveCfg = Debug|Any CPU
+		{2FE745C3-1EDF-4421-8CD7-A63EE0C1AE43}.MinSizeRel|x64.Build.0 = Debug|Any CPU
+		{2FE745C3-1EDF-4421-8CD7-A63EE0C1AE43}.Release|x64.ActiveCfg = Release|Any CPU
+		{2FE745C3-1EDF-4421-8CD7-A63EE0C1AE43}.Release|x64.Build.0 = Release|Any CPU
+		{2FE745C3-1EDF-4421-8CD7-A63EE0C1AE43}.RelWithDebInfo|x64.ActiveCfg = Debug|Any CPU
+		{2FE745C3-1EDF-4421-8CD7-A63EE0C1AE43}.RelWithDebInfo|x64.Build.0 = Debug|Any CPU
+		{FDA791E0-2B91-4AD5-B91D-CF920A4C4E38}.Debug|x64.ActiveCfg = Debug|Any CPU
+		{FDA791E0-2B91-4AD5-B91D-CF920A4C4E38}.Debug|x64.Build.0 = Debug|Any CPU
+		{FDA791E0-2B91-4AD5-B91D-CF920A4C4E38}.MinSizeRel|x64.ActiveCfg = Debug|Any CPU
+		{FDA791E0-2B91-4AD5-B91D-CF920A4C4E38}.MinSizeRel|x64.Build.0 = Debug|Any CPU
+		{FDA791E0-2B91-4AD5-B91D-CF920A4C4E38}.Release|x64.ActiveCfg = Release|Any CPU
+		{FDA791E0-2B91-4AD5-B91D-CF920A4C4E38}.Release|x64.Build.0 = Release|Any CPU
+		{FDA791E0-2B91-4AD5-B91D-CF920A4C4E38}.RelWithDebInfo|x64.ActiveCfg = Debug|Any CPU
+		{FDA791E0-2B91-4AD5-B91D-CF920A4C4E38}.RelWithDebInfo|x64.Build.0 = Debug|Any CPU
+		{B945A183-2174-4ACB-B289-8A5C546656FE}.Debug|x64.ActiveCfg = Debug|Any CPU
+		{B945A183-2174-4ACB-B289-8A5C546656FE}.Debug|x64.Build.0 = Debug|Any CPU
+		{B945A183-2174-4ACB-B289-8A5C546656FE}.MinSizeRel|x64.ActiveCfg = Debug|Any CPU
+		{B945A183-2174-4ACB-B289-8A5C546656FE}.MinSizeRel|x64.Build.0 = Debug|Any CPU
+		{B945A183-2174-4ACB-B289-8A5C546656FE}.Release|x64.ActiveCfg = Release|Any CPU
+		{B945A183-2174-4ACB-B289-8A5C546656FE}.Release|x64.Build.0 = Release|Any CPU
+		{B945A183-2174-4ACB-B289-8A5C546656FE}.RelWithDebInfo|x64.ActiveCfg = Debug|Any CPU
+		{B945A183-2174-4ACB-B289-8A5C546656FE}.RelWithDebInfo|x64.Build.0 = Debug|Any CPU
+		{39AEA075-6457-46C0-8821-1158BE6EFA57}.Debug|x64.ActiveCfg = Debug|Any CPU
+		{39AEA075-6457-46C0-8821-1158BE6EFA57}.Debug|x64.Build.0 = Debug|Any CPU
+		{39AEA075-6457-46C0-8821-1158BE6EFA57}.MinSizeRel|x64.ActiveCfg = Debug|Any CPU
+		{39AEA075-6457-46C0-8821-1158BE6EFA57}.MinSizeRel|x64.Build.0 = Debug|Any CPU
+		{39AEA075-6457-46C0-8821-1158BE6EFA57}.Release|x64.ActiveCfg = Release|Any CPU
+		{39AEA075-6457-46C0-8821-1158BE6EFA57}.Release|x64.Build.0 = Release|Any CPU
+		{39AEA075-6457-46C0-8821-1158BE6EFA57}.RelWithDebInfo|x64.ActiveCfg = Debug|Any CPU
+		{39AEA075-6457-46C0-8821-1158BE6EFA57}.RelWithDebInfo|x64.Build.0 = Debug|Any CPU
+		{9F6D7DE8-185B-4C1D-B809-3314D092DF28}.Debug|x64.ActiveCfg = Debug|Any CPU
+		{9F6D7DE8-185B-4C1D-B809-3314D092DF28}.Debug|x64.Build.0 = Debug|Any CPU
+		{9F6D7DE8-185B-4C1D-B809-3314D092DF28}.MinSizeRel|x64.ActiveCfg = Debug|Any CPU
+		{9F6D7DE8-185B-4C1D-B809-3314D092DF28}.MinSizeRel|x64.Build.0 = Debug|Any CPU
+		{9F6D7DE8-185B-4C1D-B809-3314D092DF28}.Release|x64.ActiveCfg = Release|Any CPU
+		{9F6D7DE8-185B-4C1D-B809-3314D092DF28}.Release|x64.Build.0 = Release|Any CPU
+		{9F6D7DE8-185B-4C1D-B809-3314D092DF28}.RelWithDebInfo|x64.ActiveCfg = Debug|Any CPU
+		{9F6D7DE8-185B-4C1D-B809-3314D092DF28}.RelWithDebInfo|x64.Build.0 = Debug|Any CPU
+		{6BD3F6D8-B6FC-44E8-B8FE-06352F638B9B}.Debug|x64.ActiveCfg = Debug|Any CPU
+		{6BD3F6D8-B6FC-44E8-B8FE-06352F638B9B}.Debug|x64.Build.0 = Debug|Any CPU
+		{6BD3F6D8-B6FC-44E8-B8FE-06352F638B9B}.MinSizeRel|x64.ActiveCfg = Debug|Any CPU
+		{6BD3F6D8-B6FC-44E8-B8FE-06352F638B9B}.MinSizeRel|x64.Build.0 = Debug|Any CPU
+		{6BD3F6D8-B6FC-44E8-B8FE-06352F638B9B}.Release|x64.ActiveCfg = Release|Any CPU
+		{6BD3F6D8-B6FC-44E8-B8FE-06352F638B9B}.Release|x64.Build.0 = Release|Any CPU
+		{6BD3F6D8-B6FC-44E8-B8FE-06352F638B9B}.RelWithDebInfo|x64.ActiveCfg = Debug|Any CPU
+		{6BD3F6D8-B6FC-44E8-B8FE-06352F638B9B}.RelWithDebInfo|x64.Build.0 = Debug|Any CPU
 	EndGlobalSection
 	GlobalSection(SolutionProperties) = preSolution
 		HideSolutionNode = FALSE
@@ -80,4 +204,19 @@ Global
 	GlobalSection(Performance) = preSolution
 		HasPerformanceSessions = true
 	EndGlobalSection
+	GlobalSection(NestedProjects) = preSolution
+		{C09ED521-D2DF-40BC-A76F-EB3CB870B2F6} = {267F81EF-A8AB-49FD-9F56-D7EC3C1F7C7C}
+		{EF26A475-6101-4C9C-A717-D60BB92A2302} = {C09ED521-D2DF-40BC-A76F-EB3CB870B2F6}
+		{911B08A3-515A-441D-84C8-8A43740B5FC1} = {C09ED521-D2DF-40BC-A76F-EB3CB870B2F6}
+		{14140310-2233-430A-821B-F3058A25234B} = {C09ED521-D2DF-40BC-A76F-EB3CB870B2F6}
+		{43146EA8-6FCC-46B8-88B0-42F9BB5E6E0A} = {C09ED521-D2DF-40BC-A76F-EB3CB870B2F6}
+		{6F22DB96-6EAB-4034-99D9-287021C36F01} = {C09ED521-D2DF-40BC-A76F-EB3CB870B2F6}
+		{A5C33956-A41F-49B3-A41D-1082CBA59D02} = {C09ED521-D2DF-40BC-A76F-EB3CB870B2F6}
+		{2FE745C3-1EDF-4421-8CD7-A63EE0C1AE43} = {C09ED521-D2DF-40BC-A76F-EB3CB870B2F6}
+		{FDA791E0-2B91-4AD5-B91D-CF920A4C4E38} = {C09ED521-D2DF-40BC-A76F-EB3CB870B2F6}
+		{B945A183-2174-4ACB-B289-8A5C546656FE} = {C09ED521-D2DF-40BC-A76F-EB3CB870B2F6}
+		{39AEA075-6457-46C0-8821-1158BE6EFA57} = {C09ED521-D2DF-40BC-A76F-EB3CB870B2F6}
+		{9F6D7DE8-185B-4C1D-B809-3314D092DF28} = {C09ED521-D2DF-40BC-A76F-EB3CB870B2F6}
+		{6BD3F6D8-B6FC-44E8-B8FE-06352F638B9B} = {C09ED521-D2DF-40BC-A76F-EB3CB870B2F6}
+	EndGlobalSection
 EndGlobal
@@ -0,0 +1,49 @@
+include_guard(DIRECTORY)
+
+function(protobuf_generate_fs)
+  set(_options)
+  set(_one_value_args TARGET_NAME OUT_DIR PROTOC)
+  set(_multi_value_args PROTO_INCLUDE_DIRS PROTO_SOURCES)
+  cmake_parse_arguments(_ARGS "${_options}" "${_one_value_args}" "${_multi_value_args}" ${ARGN})
+
+  if (NOT DEFINED _ARGS_TARGET_NAME)
+    set(_ARGS_TARGET_NAME "${PROJECT_NAME}_generate_fs_protos")
+  endif()
+  if (NOT DEFINED _ARGS_PROTOC)
+    set(_ARGS_PROTOC "${SPARK_BIN}/protoc")
+  endif()
+
+  set(_proto_include_cli_args)
+  foreach(_dir ${_ARGS_PROTO_INCLUDE_DIRS})
+    list(APPEND _proto_include_cli_args -I ${_dir})
+  endforeach()
+
+  set(_outputs)
+  foreach(_proto_source ${_ARGS_PROTO_SOURCES})
+    # get_filename_component(_abs_proto_source ${_proto_source} ABSOLUTE)
+    get_filename_component(_proto_source_we ${_proto_source} NAME_WE)
+    # get_filename_component(_proto_source_dir ${_proto_source} DIRECTORY)
+    list(APPEND _outputs ${_ARGS_OUT_DIR}/${_proto_source_we}.fable.fs)
+  endforeach()
+
+  set(_venv_exec ${CMAKE_SOURCE_DIR}/tools/venv_exec.sh)
+  set(_plugin_py ${CMAKE_SOURCE_DIR}/src/lib/api/service_compiler/plugin.py)
+  set(_template_fs ${CMAKE_SOURCE_DIR}/src/lib/api/service_compiler/templates/fs/fable.fs)
+  add_custom_command(
+    OUTPUT ${_outputs}
+    WORKING_DIRECTORY ${CMAKE_CURRENT_SOURCE_DIR}
+    COMMAND ${_venv_exec}
+      ${_ARGS_PROTOC}
+      --plugin=protoc-gen-service=${_plugin_py}
+      --service_out=lang=fs,strip_out_path=1:${_ARGS_OUT_DIR}
+      ${_proto_include_cli_args}
+      ${_ARGS_PROTO_SOURCES}
+    DEPENDS
+      ${_venv_exec}
+      ${_plugin_py}
+      ${_template_fs}
+      ${CMAKE_CURRENT_SOURCE_DIR}/package-lock.json
+      ${_ARGS_PROTO_SOURCES}
+  )
+  add_custom_target(${_ARGS_TARGET_NAME} ALL DEPENDS "${_outputs}")
+endfunction()
@@ -0,0 +1,33 @@
+include_guard(DIRECTORY)
+
+function(protobuf_generate_js)
+  set(_options)
+  set(_one_value_args TARGET_NAME OUTPUT)
+  set(_multi_value_args PROTO_INCLUDE_DIRS PROTO_SOURCES)
+  cmake_parse_arguments(_ARGS "${_options}" "${_one_value_args}" "${_multi_value_args}" ${ARGN})
+
+  if (NOT DEFINED _ARGS_TARGET_NAME)
+    set(_ARGS_TARGET_NAME "${PROJECT_NAME}_generate_js_protos")
+  endif()
+
+  set(_proto_include_cli_args)
+  foreach(_dir ${_ARGS_PROTO_INCLUDE_DIRS})
+    list(APPEND _proto_include_cli_args -p ${_dir})
+  endforeach()
+
+  add_custom_command(
+    OUTPUT ${_ARGS_OUTPUT}
+    WORKING_DIRECTORY ${CMAKE_CURRENT_SOURCE_DIR}
+    COMMAND npx
+      --no-install pbjs
+      -t json-module
+      -w commonjs
+      -o ${_ARGS_OUTPUT}
+      ${_proto_include_cli_args}
+      ${_ARGS_PROTO_SOURCES}
+    DEPENDS
+      ${CMAKE_CURRENT_SOURCE_DIR}/package-lock.json
+      ${_ARGS_PROTO_SOURCES}
+  )
+  add_custom_target(${_ARGS_TARGET_NAME} ALL DEPENDS ${_ARGS_OUTPUT})
+endfunction()
@@ -0,0 +1,49 @@
+include_guard(DIRECTORY)
+include(${CMAKE_SOURCE_DIR}/cmake/TernaryOperator.cmake)
+
+function(sym_link_dir)
+  set(_options)
+  set(_one_value_args TARGET_NAME SOURCE DEST COMMENT)
+  set(_multi_value_args)
+  cmake_parse_arguments(_ARGS "${_options}" "${_one_value_args}" "${_multi_value_args}" ${ARGN})
+  add_custom_command(
+    OUTPUT ${_ARGS_DEST}
+    COMMAND ln ARGS -sf ${_ARGS_SOURCE} ${_ARGS_DEST}
+  )
+  set_ternary(_comment _ARGS_COMMENT "${_ARGS_COMMENT}" "linking dir ${_ARGS_SOURCE} -> ${_ARGS_DEST}")
+  add_custom_target(
+    ${_ARGS_TARGET_NAME} ALL
+    DEPENDS ${_ARGS_DEST}
+    COMMENT "${_comment}"
+  )
+endfunction()
+
+function(sym_link_file)
+  set(_options)
+  set(_one_value_args SOURCE DEST INSTALL_DEST)
+  set(_multi_value_args)
+  cmake_parse_arguments(_ARGS "${_options}" "${_one_value_args}" "${_multi_value_args}" ${ARGN})
+  file(
+    CREATE_LINK
+      ${_ARGS_SOURCE}
+      ${_ARGS_DEST}
+    SYMBOLIC
+  )
+  if (DEFINED _ARGS_INSTALL_DEST)
+    install(
+      FILES
+      ${_ARGS_SOURCE}
+      DESTINATION ${_ARGS_INSTALL_DEST}
+    )
+  endif()
+endfunction()
+
+function(sym_link_local_lib LIB_NAME)
+  set(_lib_file_name "${CMAKE_SHARED_LIBRARY_PREFIX}${LIB_NAME}.so")
+  file(
+    CREATE_LINK
+      "${CMAKE_BINARY_DIR}/src/lib_${LIB_NAME}/${_lib_file_name}"
+      "${CMAKE_CURRENT_BINARY_DIR}/${_lib_file_name}"
+    SYMBOLIC
+  )
+endfunction()
@@ -0,0 +1,9 @@
+include_guard(GLOBAL)
+
+macro(set_ternary RESULT CONDITION TRUE_VALUE FALSE_VALUE)
+  if(${CONDITION})
+    set(${RESULT} ${TRUE_VALUE})
+  else()
+    set(${RESULT} ${FALSE_VALUE})
+  endif()
+endmacro()
@@ -0,0 +1,25 @@
+include(ExternalProject)
+
+if(NOT TARGET websocket-library_project)
+ExternalProject_Add(websocket-library_project 
+    GIT_REPOSITORY https://fintech.websocket:h_ubh-4dsWRB569wwKyg@gitlab.com/cryptoexchanges1/websocket-client-using-epoll.git
+    GIT_TAG enhancement/config-ssl
+    CMAKE_ARGS 
+              -DCMAKE_INSTALL_PREFIX=${CMAKE_CURRENT_BINARY_DIR}/websocket-library 
+              -DCMAKE_BUILD_TYPE=Release
+              -DLOG_LEVEL=INFO
+              -DENABLE_ENCRYPTION=True
+)
+endif()
+
+ExternalProject_Get_Property(websocket-library_project BINARY_DIR)
+ExternalProject_Get_Property(websocket-library_project SOURCE_DIR)
+
+set(websocket-library_INCLUDE_DIR ${SOURCE_DIR}/include) 
+set(websocket-library_LIB_DIR ${BINARY_DIR}/src) 
+
+add_library(websocket-library STATIC IMPORTED)
+
+set_property(TARGET websocket-library PROPERTY IMPORTED_LOCATION ${websocket-library_LIB_DIR}/libemumba-websocket-library.a)
+add_dependencies(websocket-library websocket-library_project)
+install(FILES ${websocket-library_LIB_DIR}/libemumba-websocket-library.a DESTINATION ${CMAKE_INSTALL_LIBDIR})
@@ -4,6 +4,8 @@ namespace bts.SilverPine {
   using System;

   public class NullFilterStream : ViewStream {
+      public NullFilterStream(ViewStream parent_str, string col_name)
+      : this(parent_str, col_name, true) { }
      public NullFilterStream(ViewStream parent_str, string col_name, bool filter_out_doublenans = false) {
         ViewStream rs = new FilterStream(parent_str,
         r => {
@@ -0,0 +1,91 @@
+namespace bts.SilverPine {
+   using BCG;
+
+   using System;
+   using System.Collections.Generic;
+   using System.Linq;
+
+   using ZstdSharp.Unsafe;
+
+   public enum GroupByBundleOp {
+      Mean,
+      Median,
+      Mode,
+      StdDev,
+      Max,
+      Min,
+      First,
+      Last,
+      Sum
+   }
+   public class GroupByBundle {
+      public readonly List<Tuple<string, GroupByBundleOp>> operations;
+
+      public static GroupByBundle parse(string text) {
+         // text is inp_col: op, inp_col: op, inp_col:op
+         List<Tuple<string, GroupByBundleOp>> rv = new List<Tuple<string, GroupByBundleOp>>();
+         string[] calcs = text.Split(',');
+         foreach (string c in calcs) {
+            string[] toks = c.Split(':');
+            string inp_col = toks[0].Trim();
+            string op_txt = toks[1].Trim();
+            GroupByBundleOp op = Enum.Parse<GroupByBundleOp>(op_txt, true);
+            rv.Add(Tuple.Create(inp_col, op));
+         }
+         return new GroupByBundle(rv);
+      }
+
+      public GroupByBundle(List<Tuple<string, GroupByBundleOp>> operations_) {
+         operations = operations_;
+      }
+
+      public static IBlueRow group_by(List<IBlueRow> rows, GroupByBundle parameters) {
+         if (rows.Count() == 0) {
+            return null;
+         }
+
+         IBlueRow ir = rows.ElementAt(0);
+         IBlueTableSchema in_schema = ir.schema;
+         IBlueTableSchema out_schema = in_schema;
+         IDictionary<string, Tuple<string, GroupByBundleOp>> out_col_map =
+            new Dictionary<string, Tuple<string, GroupByBundleOp>>();
+         foreach (var t in parameters.operations) {
+            string out_col_name = $"{t.Item1}_{t.Item2}".ToLower();
+            out_schema = out_schema.add_column(TypeCode.Double, out_col_name);
+            out_col_map[out_col_name] = t;
+         }
+         MutableBlueRow or = new MutableBlueRow(out_schema, ir);
+         foreach (var kv in out_col_map) {
+            string out_col_name = kv.Key;
+            string in_col_name = kv.Value.Item1;
+            GroupByBundleOp op = kv.Value.Item2;
+            IEnumerable<double> col_vals = BlueDataUtility.get_colvalues<double>(rows, in_col_name);
+            double comp_res = compute_op(col_vals, op);
+            or.setValue(out_col_name, comp_res);
+         }
+         return or;
+      }
+
+      public static double compute_op(IEnumerable<double> col_vals, GroupByBundleOp op) {
+         switch (op) {
+            case GroupByBundleOp.Max:
+               return col_vals.Max<double>();
+            case GroupByBundleOp.Min:
+               return col_vals.Min<double>();
+            case GroupByBundleOp.Mean:
+               return col_vals.Sum() / col_vals.Count();
+            case GroupByBundleOp.Sum:
+               return col_vals.Sum();
+            case GroupByBundleOp.First:
+               return col_vals.First();
+            case GroupByBundleOp.Last:
+               return col_vals.Last();
+            default:
+               // TODO
+               Log.assert(false);
+               return Double.NaN;
+         }
+      }
+
+   }
+}
@@ -2,10 +2,13 @@
 using System.Collections.Generic;
 using System.Diagnostics;
 using System.Linq;
+using System.Text.Json;

 using bts;
+using bts.SilverPine;

 namespace BCG {
+   [DataStreamUserRespresentation("aggregate")]
   public class NonIncGroupByStream : ViewStream {
      private class DelNonIncGroupByEvaluator : INonIncGroupByEvaluator {
         Func<string[]> _groupby_cols_func;
@@ -43,13 +46,31 @@ namespace BCG {
         _main_stream = main_stream;
         _groupby_func = groupby_func;
         _groupby_cols = _groupby_func.get_group_by_cols();
-         _row_hasher = new BlueRowHasher(_groupby_cols.Length);
+         _row_hasher = new BlueRowHasher(_groupby_cols == null ? 0 : _groupby_cols.Length);
      }

      public NonIncGroupByStream(ViewStream main_stream, Func<string[]> groupby_cols_func,
                                 Func<List<IBlueRow>, List<IBlueRow>> evaluate_func)
                                 : this(main_stream, new DelNonIncGroupByEvaluator(groupby_cols_func, evaluate_func)) { }

+      public NonIncGroupByStream(ViewStream parent_str, string groupby_col, string standard_group_by_text, string optional_sort_col = null) {
+         GroupByBundle std_group_by = GroupByBundle.parse(standard_group_by_text);
+         Func<string[]> groupby_cols_func = () => {
+            return String.IsNullOrEmpty(groupby_col) ? null : new string[] { groupby_col };
+         };
+         Func<List<IBlueRow>, List<IBlueRow>> evaluate_func = (group_rows) => {
+            if (!String.IsNullOrEmpty(optional_sort_col))
+               group_rows.Sort((r1, r2) => BlueDataUtility.compare_rows(r2, r1, optional_sort_col));
+            IBlueRow rv = GroupByBundle.group_by(group_rows, std_group_by);
+            return new IBlueRow[] { rv }.ToList();
+         };
+         ViewStream rs = new NonIncGroupByStream(parent_str, groupby_cols_func, evaluate_func);
+         mySetRealSource(rs);
+      }
+      public NonIncGroupByStream(ViewStream parent_str, string standard_group_by_text) :
+      this(parent_str, String.Empty, standard_group_by_text) {
+      }
+

      protected override void myOnFirstSubscriber() { _main_stream.onChange += _main_stream_onchange; }

@@ -79,7 +100,9 @@ namespace BCG {
            Log.DataFlow.info(
               $"Time to evaluate {grouped_rows.Count} groups {_timer.ElapsedMilliseconds}ms, ratio {_n_expensive_fires}/{_n_total_fires}");
         }
-         myFireEvent(new DataPacket(total_state, total_state, myLastValue == null ? new BlueTable(total_state.schema) : (myLastValue as DataPacket).newRows));
+         // TODO,
+         if (total_state != null)
+            myFireEvent(new DataPacket(total_state, total_state, myLastValue == null ? new BlueTable(total_state.schema) : (myLastValue as DataPacket).newRows));
      }
   }
 }
@@ -1,10 +1,12 @@
 using bts;
+using bts.collections;

 using mbts;

 using System;
 using System.Collections.Generic;
 using System.Diagnostics;
+using System.Linq;
 using System.Security.Cryptography;

 namespace BCG {
@@ -35,6 +37,38 @@ namespace BCG {
         _is_tag_along = _join_spec.join_col_names.Length == 0;
      }

+      private bool _filled_missing_data = false;
+      private void _fill_missing_data(DataPacket dp) {
+         if (_filled_missing_data)
+            return;
+         Func<string[], bool> null_or_empty = (str_arr) => {
+            if (str_arr == null || str_arr.Length == 0)
+               return true;
+            foreach (string s in str_arr) {
+               if (!String.IsNullOrEmpty(s)) {
+                  return false;
+               }
+            }
+            return true;
+         };
+         Func<string[], string[]> remove_bad_cols = (col_names) => {
+            HashSet<string> schema_col_map = new HashSet<string>(dp.schema.columnNames);
+            List<string> cols_list = new List<string>(col_names);
+            cols_list.RemoveWhere(c => !schema_col_map.Contains(c));
+            return cols_list.ToArray();
+         };
+         if (null_or_empty(_join_spec.data_col_names)) {
+            _join_spec.data_col_names = dp.schema.columnNames;
+         }
+         _join_spec.data_col_names = remove_bad_cols(_join_spec.data_col_names);
+         if (null_or_empty(_join_spec.data_col_names)) {
+            _join_spec.data_col_names = dp.schema.columnNames;
+         }
+         if (null_or_empty(_join_spec.projected_col_names)) {
+            _join_spec.projected_col_names = _join_spec.data_col_names;
+         }
+      }
+
      public void init() {
         _join_spec.stream.onChange += _on_stream_cb;
      }
@@ -43,6 +77,7 @@ namespace BCG {
         Log.Gen.info($"NonIncJoinBookkeeper._on_stream_cb dp_obj: {dp_obj}");
         var timer = Stopwatch.StartNew();
         DataPacket dp = (DataPacket)dp_obj;
+         _fill_missing_data(dp);
         if (_schema == null) {
            _schema = dp.schema;
            _join_col_indices = BlueDataUtility.get_column_indices(_schema, _join_spec.join_col_names);
@@ -7,11 +7,12 @@ using System.Threading.Tasks;
 using mbts;

 namespace BCG {
+   // this a collection of data, theres no joining logic in here
   public class NonIncJoinSpec {
      public ViewStream stream;
      // join col names are the names on which the streams are joined (for equality),
      // data col names are the names in the original stream
-      // projected col naes are their new names, ie if its x:l_x, then x is in the 
+      // projected col names are their new names, ie if its x:l_x, then x is in the 
      // data_col_names and l_x in projected cols
      public string[] join_col_names, data_col_names, projected_col_names;

@@ -1,9 +1,8 @@
-using Google.Protobuf.WellKnownTypes;
-
 using System;
-using BCG.Utilities.General;

-namespace BCG {
+using BCG;
+
+namespace bts.SilverPine {
   // Keeps adding changing rows to the stream, hence preserving the history
   // and creating a time series, for example this can be used to make bbo updates
   // into a time series of prices/qty 
@@ -9,161 +9,16 @@ using BCG;
 using System.Linq;
 using System.Data.Entity;
 using System.Collections.Generic;
+using System.Data.SQLite;
+

 namespace bts.SilverPine {

-   public class DataRowWrapperRow : IBlueRow {
-      private readonly DataRow _row;
-      private readonly IBlueTableSchema _schema;
-      private readonly int _id;
-      public DataRowWrapperRow(IBlueTableSchema schema, DataRow row, int id) {
-         _row = row;
-         _id = id;
-         _schema = schema;
-      }
-      public virtual IBlueTableSchema schema { get { return _schema; } }
-
-      public virtual object this[string col_name] {
-         get {
-            ColumnInfo ci = _schema[col_name];
-            return this[ci.colNo];
-         }
-      }
-      public virtual object this[int index] {
-         get {
-            if (_schema[index].colName == "id")
-               return _id;
-            return _row[index];
-         }
-      }
-
-      public virtual int Count { get { return _row.ItemArray.Length; } }
-
-      public virtual bool IsSynchronized { get { return false; } }
-
-      public virtual object SyncRoot { get { return this; } }
-
-      public virtual void CopyTo(Array array, int index) { _row.ItemArray.CopyTo(array, index); }
-
-      public virtual bool IsFixedSize { get { return true; } }
-
-      virtual public bool IsReadOnly { get { return true; } }
-
-      /// <summary>
-      /// The key can either be the string corrsponding to the column name or
-      /// the order of the column
-      /// </summary>
-      public virtual object this[object key] {
-         get {
-            if (key is string)
-               return this[(string)key];
-            else if (key is int)
-               return this[(int)key];
-            throw new BlueDataException("Wrong type while indexing the row");
-         }
-         set { throw new NotSupportedException("The IBlueRow is readonly"); }
-      }
-
-      public virtual ICollection Keys { get { return _schema.columnNames; } }
-
-      public virtual ICollection Values { get { return _row.ItemArray; } }
-
-      public virtual void Add(object key, object value) { throw new NotSupportedException("The IBlueRow is readonly"); }
-
-      public virtual void Clear() { throw new NotSupportedException("The IBlueRow is readonly"); }
-
-      public virtual bool Contains(object key) { return _schema[(string)key] != null; }
-
-      public IDictionaryEnumerator GetEnumerator() { return new BlueRowDictionaryEnumerator(this); }
-
-      IEnumerator IEnumerable.GetEnumerator() { return new BlueRowEnumerator(this); }
-
-      public virtual void Remove(object key) { throw new NotSupportedException("The IBlueRow is readonly"); }
-
-   }
-
-   public class DataTableWrapperTable : IBlueRowCollection {
-
-      public class DataTableWrapperTableEnumerator : IEnumerator {
-         private IEnumerator _enumerator;
-         private IBlueTableSchema _schema;
-         private int _id = 0;
-         public DataTableWrapperTableEnumerator(DataTable table) {
-            _enumerator = table.Rows.GetEnumerator();
-            _schema = BlueDataUtility.getSchema(table);
-
-         }
-         public object Current {
-            get {
-               DataRow r = _enumerator.Current as DataRow;
-               return new DataRowWrapperRow(_schema, r, _id);
-            }
-         }
-         public bool MoveNext() {
-            _id++;
-            return _enumerator.MoveNext();
-         }
-         public void Reset() {
-            _id = 0;
-            _enumerator.Reset();
-         }
-      }
-
-      private readonly DataTable _table;
-      private IBlueTableSchema _schema;
-      private IEnumerator _enumerator;
-
-      public DataTableWrapperTable(DataTable table) {
-         _table = table;
-         _schema = BlueDataUtility.getSchema(_table);
-         _enumerator = new DataTableWrapperTableEnumerator(table);
-      }
-      public virtual IBlueRow this[int id] {
-         get {
-            // its assumed the rows index == id
-            DataRow dr = _table.Rows[id];
-            return new DataRowWrapperRow(_schema, dr, id);
-         }
-         set {
-
-         }
-      }
-
-      public virtual IBlueTableSchema schema { get { return _schema; } }
-
-      public virtual bool containsID(int id) { return id < _table.Rows.Count; }
-
-      // private List<IBlueRow> _rows() {
-      //    List<IBlueRow> rv = new List<IBlueRow>();
-      //    int id = 0;
-      //    foreach (DataRow r in _table.Rows) {
-      //       rv.Add(new DataRowWrapperRow(r, id++));
-      //    }
-      //    return rv;
-      // }
-
-      public virtual IEnumerator GetEnumerator() {
-         // var rows = _rows();
-         // return rows.GetEnumerator();
-         return _enumerator;
-      }
-
-      public virtual bool IsSynchronized { get { return false; } }
-
-      public virtual object SyncRoot { get { return this; } }
-
-      public virtual void CopyTo(Array array, int index) {
-         Log.assert(false);
-         //_rows().CopyTo(array as IBlueRow[], index); 
-      }
-
-      public virtual int Count {
-         get { return _table.Rows.Count; }
-      }
-   }
-
   public static class SQLiteHelper {
-      public static DataTable ExecuteSelectQuery(string connection_string, string query) {
+      public static string location = "C:\\Users\\m_tah\\development\\data\\";
+      public static string db_name = "stream_data.db";
+
+      public static DataTable execute_select_query(string connection_string, string query) {
         // Create a new DataTable to store the results
         DataTable dataTable = new DataTable();

@@ -183,48 +38,85 @@ namespace bts.SilverPine {
               Console.WriteLine("An error occurred: " + ex.Message);
            }
         }
-
         // Return the populated DataTable
         return dataTable;
      }
-   }

-   [DataStreamUserRespresentation("SQLite")]
-   public class SQLiteStream : ViewStream {
-      private readonly string _connection_string, _query;
-      private Stopwatch _stopwatch = new Stopwatch();
-      public SQLiteStream(string connection_string, string query) {
-         _connection_string = connection_string;
-         _query = query;
+      public static List<Dictionary<string, object>> ExecuteBulkRead(string connectionString, string query) {
+         var result = new List<Dictionary<string, object>>();
+
+         using (var connection = new SQLiteConnection(connectionString)) {
+            connection.Open();
+
+            // Begin a transaction to optimize read speed
+            using (var transaction = connection.BeginTransaction())
+            using (var command = new SQLiteCommand(query, connection, transaction))
+            using (var reader = command.ExecuteReader()) {
+               while (reader.Read()) {
+                  var row = new Dictionary<string, object>();
+
+                  for (int i = 0; i < reader.FieldCount; i++) {
+                     string columnName = reader.GetName(i);
+                     object columnValue = reader.IsDBNull(i) ? null : reader.GetValue(i);
+                     row[columnName] = columnValue;
+                  }
+
+                  result.Add(row);
+               }
+
+               // Commit the transaction
+               transaction.Commit();
+            }
+         }
+
+         return result;
      }

+      public static List<Dictionary<string, object>> get_table(string connectionString, string query) {
+         var result = new List<Dictionary<string, object>>();

-      private void _read_from_sqlite_nd_fire() {
-         _stopwatch.Start();
-         DataTable dt = SQLiteHelper.ExecuteSelectQuery(_connection_string, _query);
-         _stopwatch.Stop();
-         Log.Gen.info($"SQLiteStream._read_from_sqlite_and_fire time to read db: {_stopwatch.ElapsedMilliseconds} msecs");
-         _stopwatch.Start();
-         IBlueRowCollection rows = BCG.BlueDataUtility.toIBlueRowCollection(dt);
-         //IBlueRowCollection rows = new DataTableWrapperTable(dt);
-         _stopwatch.Stop();
-         Log.Gen.info($"SQLiteStream._read_from_sqlite_and_fire time to load into IBlueRowCollection: {_stopwatch.ElapsedMilliseconds} msecs");
-         DataPacket dp = BlueDataUtility.build_dp_inc(rows, myLastValue as DataPacket);
-         myFireEvent(dp);
+         using (var connection = new SQLiteConnection(connectionString)) {
+            try {
+               connection.Open();
+
+               using (var command = new SQLiteCommand(query, connection))
+               using (var reader = command.ExecuteReader()) {
+                  while (reader.Read()) {
+                     var row = new Dictionary<string, object>();
+
+                     for (int i = 0; i < reader.FieldCount; i++) {
+                        string columnName = reader.GetName(i);
+                        object columnValue = reader.IsDBNull(i) ? null : reader.GetValue(i);
+                        row[columnName] = columnValue;
+                     }
+
+                     result.Add(row);
+                  }
+               }
+            } catch (Exception ex) {
+               Console.WriteLine("An error occurred: " + ex.Message);
+            }
+         }
+         return result;
      }

-      protected override void myOnFirstSubscriber() {
-         _read_from_sqlite_nd_fire();
+      public static IEnumerable<Dictionary<string, object>> get_table_2(string connectionString, string query) {
+         using (var connection = new SQLiteConnection(connectionString)) {
+            connection.Open();
+            using (var command = new SQLiteCommand(query, connection))
+            using (var reader = command.ExecuteReader()) {
+               while (reader.Read()) {
+                  var row = new Dictionary<string, object>();
+                  for (int i = 0; i < reader.FieldCount; i++) {
+                     string columnName = reader.GetName(i);
+                     object columnValue = reader.IsDBNull(i) ? null : reader.GetValue(i);
+                     row[columnName] = columnValue;
+                  }
+                  yield return row;
+               }
+            }
+         }
      }
-
-      protected override void myOnLastSubscriber() {
-         // this value will be cleared by the base class
-         // but in this particular stream this behaviour
-         // is not usefull since any new subscribers 
-         // would force the query to be recreated
-         //myLastDP = myLastValue;
-      }
-
   }
 }

@@ -0,0 +1,73 @@
+namespace bts.SilverPine {
+   using System;
+   using System.Data;
+   using System.Data.SQLite;
+   using System.Diagnostics;
+   using System.Collections;
+
+
+   using BCG;
+
+   [DataStreamUserRespresentation("SQLite")]
+   public class SQLiteStream : ViewStream {
+      private readonly string _connection_string, _query;
+      private Stopwatch _stopwatch = new Stopwatch();
+
+      private SQLiteStream(string connection_string, string query, bool noop) {
+         _connection_string = connection_string;
+         _query = query;
+      }
+
+      public SQLiteStream(string db_name, string query)
+      :
+      this($"Data Source={SQLiteHelper.location}{db_name};Version=3", query, true) { }
+
+      public SQLiteStream(string query)
+      :
+      this(SQLiteHelper.db_name, query) {
+      }
+
+
+      private void _read_from_sqlite_and_fire() {
+         _stopwatch.Start();
+         DataTable dt = SQLiteHelper.execute_select_query(_connection_string, _query);
+         _stopwatch.Stop();
+         Log.Gen.info($"SQLiteStream.ExecuteSelectQuery time to read db: {_stopwatch.ElapsedMilliseconds} msecs");
+         _stopwatch.Restart();
+         #region perf measures
+         // SQLiteHelper.get_table(_connection_string, _query);
+         // _stopwatch.Stop();
+         // Log.Gen.info($"SQLiteStream.get_table time to read db: {_stopwatch.ElapsedMilliseconds} msecs");
+         // _stopwatch.Restart();
+         // SQLiteHelper.ExecuteBulkRead(_connection_string, _query);
+         // _stopwatch.Stop();
+         // Log.Gen.info($"SQLiteStream.ExecuteBulkRead time to read db: {_stopwatch.ElapsedMilliseconds} msecs");
+         // _stopwatch.Restart();
+         // SQLiteHelper.get_table_2(_connection_string, _query);
+         // _stopwatch.Stop();
+         // Log.Gen.info($"SQLiteStream.get_table_2 time to read db: {_stopwatch.ElapsedMilliseconds} msecs");
+         // _stopwatch.Restart();
+         #endregion
+         IBlueRowCollection rows = BCG.BlueDataUtility.toIBlueRowCollection(dt);
+         //IBlueRowCollection rows = new DataTableWrapperTable(dt);
+         _stopwatch.Stop();
+         Log.Gen.info($"SQLiteStream._read_from_sqlite_and_fire time to load into IBlueRowCollection: {_stopwatch.ElapsedMilliseconds} msecs");
+         DataPacket dp = BlueDataUtility.build_dp_inc(rows, myLastValue as DataPacket);
+         myFireEvent(dp);
+      }
+
+      protected override void myOnFirstSubscriber() {
+         _read_from_sqlite_and_fire();
+      }
+
+      protected override void myOnLastSubscriber() {
+         // this value will be cleared by the base class
+         // but in this particular stream this behaviour
+         // is not usefull since any new subscribers 
+         // would force the query to be recreated
+         //myLastDP = myLastValue;
+      }
+
+   }
+
+}
@@ -80,7 +80,7 @@ namespace BCG {

      public IBlueRow map(IBlueRow row) {
         MutableBlueRow rv = new MutableBlueRow(getOutputSchema(row.schema), row);
-         rv.setValue(myConversionColName, mapDateTime(row[myConversionColName], myConversion));
+         rv.setValue(myConversionColName, mapDateTime(row.get_datetime(myConversionColName), myConversion));
         return rv;
         //return BlueDataUtility.getNewReplacedRow(row, mapDateTime(row[myConversionColName], myConversion), myConversionColName);
      }
@@ -10,9 +10,11 @@ namespace bts.SilverPine {
         Func<IBlueTableSchema, IBlueTableSchema> get_schema = (s) => {
            IBlueTableSchema rv = s.add_column(System.TypeCode.DateTime, $"{inp_col}_date");
            rv = rv.add_column(System.TypeCode.String, $"{inp_col}_day");
-            rv = rv.add_column(System.TypeCode.String, $"{inp_col}_month");
-            rv = rv.add_column(System.TypeCode.String, $"{inp_col}_hour");
-            rv = rv.add_column(System.TypeCode.String, $"{inp_col}_min");
+            rv = rv.add_column(System.TypeCode.Int32, $"{inp_col}_day_of_month");
+            rv = rv.add_column(System.TypeCode.Int32, $"{inp_col}_month");
+            rv = rv.add_column(System.TypeCode.Int32, $"{inp_col}_hour");
+            rv = rv.add_column(System.TypeCode.Int32, $"{inp_col}_min");
+            rv = rv.add_column(System.TypeCode.Int32, $"{inp_col}_seconds_of_day");
            return rv;
         };
         ViewStream rs = new MapStream(parent_stream,
@@ -23,9 +25,11 @@ namespace bts.SilverPine {
            MutableBlueRow rv = new MutableBlueRow(s, r);
            rv.setValue($"{inp_col}_date", dt.Date);
            rv.setValue($"{inp_col}_day", dt.DayOfWeek.ToString());
-            rv.setValue($"{inp_col}_month", dt.Date.Month.ToString());
+            rv.setValue($"{inp_col}_day_of_month", dt.Day);
+            rv.setValue($"{inp_col}_month", dt.Date.Month);
            rv.setValue($"{inp_col}_hour", dt.Hour);
            rv.setValue($"{inp_col}_min", dt.Minute);
+            rv.setValue($"{inp_col}_seconds_of_day", (dt - dt.Date).TotalSeconds);
            return rv;
         });
         mySetRealSource(rs);
@@ -39,7 +39,7 @@ namespace bts.SilverPine {
         mySetRealSource(rs);
      }

-      // for each row, does a row[new_ol_name] = row[col1] - rows[row_ffset][col2]
+      // for each row, does a row[new_col_name] = row[col1] - rows[row_ffset][col2]
      // we should generlize this for any math operation
      public ColumnDifferenceStream(ViewStream parent_stream, string sort_col_name, int row_offset, string col1, string col2, string diff_col_name, double optional_perc_diff_filter = 0.0) {
         // use GlobalMapStream
@@ -91,19 +91,32 @@ namespace bts.SilverPine {
         DataPacket dp = dp_obj as DataPacket;
         List<IBlueRow> rows = dp.newRows.to_list();
         var f1_rows = rows.Where(r => _f1(r));
-         Log.assert(f1_rows.Count() == 1);
+         //Log.assert(f1_rows.Count() == 1);
         var f2_rows = rows.Where(r => _f2(r));
-         Log.assert(f2_rows.Count() == 1);
-         IBlueRow f1_r = f1_rows.ElementAt(0);
-         IBlueRow f2_r = f2_rows.ElementAt(0);
-         double diff = f1_r.get<double>(_c1) - f2_r.get<double>(_c2);
-         IBlueTableSchema out_schema = f1_r.schema.add_columns(Tuple.Create(typeof(double), "diff"));
-         BlueTable out_table = new BlueTable(out_schema);
-         MutableBlueRow out_row = new MutableBlueRow(out_schema, f1_r);
-         out_row.setValue("diff", diff);
-         out_table.addRow(out_row);
-         DataPacket dp_out = BlueDataUtility.build_dp(out_table, myLastValue as DataPacket);
-         myFireEvent(dp_out);
+         //Log.assert(f2_rows.Count() == 1);
+         if (f1_rows != null && f1_rows.Count() > 0 && f2_rows != null && f2_rows.Count() > 0) {
+            IBlueRow f1_r = f1_rows.ElementAt(0);
+            IBlueRow f2_r = f2_rows.ElementAt(0);
+            double diff = f1_r.get<double>(_c1) - f2_r.get<double>(_c2);
+            IBlueTableSchema out_schema = f1_r.schema.add_columns(Tuple.Create(typeof(double), "diff"));
+            BlueTable out_table = new BlueTable(out_schema);
+            MutableBlueRow out_row = new MutableBlueRow(out_schema, f1_r);
+            out_row.setValue("diff", diff);
+            out_table.addRow(out_row);
+            DataPacket dp_out = BlueDataUtility.build_dp(out_table, myLastValue as DataPacket);
+            myFireEvent(dp_out);
+         } else {
+            IBlueRow f1_r = rows.ElementAt(0);
+            IBlueRow f2_r = rows.ElementAt(0);
+            double diff = Double.NaN;
+            IBlueTableSchema out_schema = f1_r.schema.add_columns(Tuple.Create(typeof(double), "diff"));
+            BlueTable out_table = new BlueTable(out_schema);
+            MutableBlueRow out_row = new MutableBlueRow(out_schema, f1_r);
+            out_row.setValue("diff", diff);
+            out_table.addRow(out_row);
+            DataPacket dp_out = BlueDataUtility.build_dp(out_table, myLastValue as DataPacket);
+            myFireEvent(dp_out);
+         }

      }
   }
@@ -18,6 +18,7 @@ using System.Text.Json.Nodes;
 using Newtonsoft.Json;
 using BCG;
 using System.Runtime.CompilerServices;
+using bts;

 namespace bts {

@@ -207,6 +208,10 @@ namespace BCG {
         return rv;
      }

+      public static bool contains_column(this IBlueTableSchema input_schema, string col) {
+         return input_schema.columnNames.Contains(col);
+      }
+
      public static IBlueTableSchema add_columns(this IBlueTableSchema input_schema, params Tuple<Type, string>[] cols) {
         List<ColumnInfo> cis = new List<ColumnInfo>(input_schema.columnInfo);
         foreach (Tuple<Type, string> c in cols)
@@ -1719,8 +1724,10 @@ namespace BCG {

      public static int[] get_column_indices(IBlueTableSchema schema, string[] col_names) {
         List<int> rv = new List<int>();
-         foreach (string c in col_names)
+         foreach (string c in col_names) {
+            Log.assert(schema.contains_column(c), $"Could not find col: {c} in table: {schema.tableName}");
            rv.Add(schema[c].colNo);
+         }
         return rv.ToArray();
      }

@@ -2207,23 +2214,53 @@ namespace BCG {
         try {
            if (v == null || v == DBNull.Value)
               return error_val;
-            else {
-               if (v is DateTime)
-                  return (DateTime)v;
-               else if (v is string) {
-                  DateTime rv_dt = error_val;
-                  DateTime.TryParse((string)v, out rv_dt);
+
+            if (v is DateTime) {
+               return (DateTime)v;
+            } else if (v is string) {
+               DateTime rv_dt;
+
+               // Try parsing the string as a DateTime
+               if (DateTime.TryParse((string)v,
+                                     System.Globalization.CultureInfo.InvariantCulture,
+                                     System.Globalization.DateTimeStyles.RoundtripKind,
+                                     out rv_dt)) {
                  return rv_dt;
               } else {
-                  double oa_date = (double)Convert.ChangeType(v, typeof(Double));
-                  return DateTime.FromOADate(oa_date);
+                  return error_val; // Return the error value if parsing fails
               }
+            } else {
+               // Assume the input can be converted to a double and use FromOADate
+               double oa_date = (double)Convert.ChangeType(v, typeof(Double));
+               return DateTime.FromOADate(oa_date);
            }
         } catch {
            return error_val;
         }
      }

+
+      // private static DateTime _get_datetime_value(object v, DateTime error_val) {
+      //    try {
+      //       if (v == null || v == DBNull.Value)
+      //          return error_val;
+      //       else {
+      //          if (v is DateTime)
+      //             return (DateTime)v;
+      //          else if (v is string) {
+      //             DateTime rv_dt = error_val;
+      //             DateTime.TryParse((string)v, out rv_dt);
+      //             return rv_dt;
+      //          } else {
+      //             double oa_date = (double)Convert.ChangeType(v, typeof(Double));
+      //             return DateTime.FromOADate(oa_date);
+      //          }
+      //       }
+      //    } catch {
+      //       return error_val;
+      //    }
+      // }
+
      #endregion

      #region CSV methods
@@ -47,6 +47,11 @@ namespace BCG {
      /// </summary>
      /// <param name="assemblyName"></param>
      public void loadAssembly(string assemblyName) {
+         Func<Type, bool> is_silverpine_only = (t) => {
+            if (!assemblyName.Contains("SharedStreams.dll"))
+               return true;
+            return t.Namespace == "bts.SilverPine";
+         };
         bts.Log.Gen.info($"PrimitiveManager.loadAssembly assemblyName: {assemblyName}");
         if (myAssemblyNames.Contains(assemblyName))
            return;
@@ -56,7 +61,7 @@ namespace BCG {
            Assembly.LoadFrom(assemblyName);

         foreach (Type type in loadedAssembly.GetTypes())
-            if (!type.IsInterface
+            if (!type.IsInterface && is_silverpine_only(type)
               && !type.IsAbstract
               && type.IsSubclassOf(typeof(ViewStream))
               && type.IsPublic) {
@@ -1,60 +1,62 @@
-"user": "bts",
-"pubsub": {
-  "milliseconds_between_sub_info_updates": 1000,
-  "milliseconds_between_time_slice_updates": 5000,
-  "milliseconds_between_no_buffer_space_retries": 100,
-  "retries_on_no_buffer_space": 1,
-  "do_it_all_seconds_between_logs": 0,
-  "heartbeat_interval": 15000,
-  "missing_heartbeats_before_gc": 50,
-  "use_managed_sockets": true,
-  "state_update_period": 5000,
-  "statistics_update_period": 5000,
-  "disposed_connection_check_period": 1000,
-  "message_processing_warning_threshold": 1000,
-  "server_port": 12345,
-  "publisher_routes": [
-    {
-      "name": "default",
-      "server_address": "bts-cloud://192.168.1.80:12345",
-      "tag": ".*"
-    }
-  ],
-  "subscriber_routes": [
-    {
-      "name": "default",
-      "server_address": "bts-cloud://192.168.1.80:12345",
-      "tag": ".*"
-    }
-  ],
-  "max_packet_size": 500000000,
-  "blue_comm_must_die": true,
-  "crushable_tags": [],
-  "tag_type_to_assembly_map": [
-    {
-      "handler_class": "BCG.PubSubServerTagHandler",
-      "assembly": "DataStreamCore.dll"
-    }
-  ],
-  "server_queue": "PubSubQueue",
-  "port_range": {
-    "first": 30400,
-    "last": 31700
-  },
-  "bind_to_random_port": false,
-  "debug_tags": "^NOTAGS$",
-  "latency_report_threshold_ms": 1500,
-  "use_substations": false,
-  "use_dynamic_clients": false,
-  "is_substation": false,
-  "internally_publish_sub_info": true,
-  "use_managed_clients_in_native_code": false,
-  "compressed_tags": "^NOTAGS$",
-  "tags_to_report_compression_ratio": "^NOTAGS$",
-  "disable_msmq": false,
-  "use_prepublished_server_state": false,
-  "ip_translation": null,
-  "gc_activation_threshold": 0,
-  "wait_period_after_gc": 1000
-}
+{
+  "user": "bts",
+  "sparkd": "http://192.168.1.80:8000/config",
+  "pubsub": {
+    "milliseconds_between_sub_info_updates": 1000,
+    "milliseconds_between_time_slice_updates": 5000,
+    "milliseconds_between_no_buffer_space_retries": 100,
+    "retries_on_no_buffer_space": 1,
+    "do_it_all_seconds_between_logs": 0,
+    "heartbeat_interval": 15000,
+    "missing_heartbeats_before_gc": 50,
+    "use_managed_sockets": true,
+    "state_update_period": 5000,
+    "statistics_update_period": 5000,
+    "disposed_connection_check_period": 1000,
+    "message_processing_warning_threshold": 1000,
+    "server_port": 12345,
+    "publisher_routes": [
+      {
+        "name": "default",
+        "server_address": "bts-cloud://192.168.1.80:12345",
+        "tag": ".*"
+      }
+    ],
+    "subscriber_routes": [
+      {
+        "name": "default",
+        "server_address": "bts-cloud://192.168.1.80:12345",
+        "tag": ".*"
+      }
+    ],
+    "max_packet_size": 500000000,
+    "blue_comm_must_die": true,
+    "crushable_tags": [],
+    "tag_type_to_assembly_map": [
+      {
+        "handler_class": "BCG.PubSubServerTagHandler",
+        "assembly": "DataStreamCore.dll"
+      }
+    ],
+    "server_queue": "PubSubQueue",
+    "port_range": {
+      "first": 30400,
+      "last": 31700
+    },
+    "bind_to_random_port": false,
+    "debug_tags": "^NOTAGS$",
+    "latency_report_threshold_ms": 1500,
+    "use_substations": false,
+    "use_dynamic_clients": false,
+    "is_substation": false,
+    "internally_publish_sub_info": true,
+    "use_managed_clients_in_native_code": false,
+    "compressed_tags": "^NOTAGS$",
+    "tags_to_report_compression_ratio": "^NOTAGS$",
+    "disable_msmq": false,
+    "use_prepublished_server_state": false,
+    "ip_translation": null,
+    "gc_activation_threshold": 0,
+    "wait_period_after_gc": 1000
+  }
 }
@@ -9,8 +9,90 @@ namespace bts.SilverPine {
   using ZstdSharp.Unsafe;

   public static class ContractUtils {
+      public class FutureTradableContract {
+         public string symbol;
+         public DateTime expiration;
+         public Exchange exchange;
+         public string contract_text;
+      }
+      public class FutureSpreadContract {
+         public List<Tuple<double, bts.SilverPine.ContractUtils.FutureTradableContract>> legs;
+         public string symbol {
+            get {
+               return legs == null || legs.Count == 0 ? String.Empty : legs.ElementAt(0).Item2.symbol;
+            }
+         }
+         public string contract_text;
+      }
+      public class FutureContinousContract {
+         public string symbol;
+         public Exchange exchange;
+      }
+      // this is for all future conytracts belonging to a symbol
+      public class FutureWildcardContract {
+         public string symbol;
+         public Exchange exchange;
+      }
+      public class FutureContract {
+         public FutureWildcardContract wildcard_contract = null;
+         public FutureTradableContract future_tradable_contract = null;
+         public FutureSpreadContract spread_tradable_contract = null;
+         public FutureContinousContract continous_constract = null;
+
+         public FutureContract(string txt) {
+            wildcard_contract = parse_wildcard_contract(txt);
+            if (wildcard_contract != null)
+               return;
+            FutureTradableContract fc = parse_future_contract(txt);
+            if (fc != null) {
+               future_tradable_contract = fc;
+               return;
+            }
+            FutureSpreadContract sc = parse_future_spread_contract(txt);
+            if (sc != null) {
+               spread_tradable_contract = sc;
+               return;
+            }
+            // this is not totally true, since ParseFutureSpreadContract does not
+            // parse ICS spreads
+            continous_constract = new FutureContinousContract() { symbol = txt };
+         }
+         public bool is_continous_contract => continous_constract != null;
+         public string symbol() {
+            if (wildcard_contract != null)
+               return wildcard_contract.symbol;
+            if (continous_constract != null) {
+               return continous_constract.symbol;
+            } else if (future_tradable_contract != null) {
+               return future_tradable_contract.symbol;
+            } else if (spread_tradable_contract != null) {
+               return spread_tradable_contract.symbol;
+            }
+            Log.assert(false);
+            return String.Empty;
+         }
+         public string contract_text {
+            get {
+               if (future_tradable_contract != null)
+                  return future_tradable_contract.contract_text;
+               if (spread_tradable_contract != null)
+                  return spread_tradable_contract.contract_text;
+               return null;
+            }
+         }
+      }
      #region Parsing
-      public static FutureContract ParseFutureContract(string contract_txt, Exchange exchange = Exchange.CME) {
+      public static FutureWildcardContract parse_wildcard_contract(string contract_txt, Exchange exchange = Exchange.CME) {
+         if (contract_txt.EndsWith("*")) {
+            contract_txt = contract_txt.Replace("*", String.Empty);
+            return new FutureWildcardContract() {
+               symbol = contract_txt,
+               exchange = exchange
+            };
+         }
+         return null;
+      }
+      public static FutureTradableContract parse_future_contract(string contract_txt, Exchange exchange = Exchange.CME) {
         // Validate the future string length
         if (!contract_txt.All(c => Char.IsLetterOrDigit(c)))
            return null;
@@ -56,23 +138,24 @@ namespace bts.SilverPine {

         // Create the FutureContract instance and set its properties
         DateTime expiration = new DateTime(year, month, 1);
-         var futureContract = new FutureContract {
+         var futureContract = new FutureTradableContract {
            symbol = symbol,
            expiration = expiration,
-            exchange = exchange
+            exchange = exchange,
+            contract_text = contract_txt
         };

         return futureContract;
      }
      // Method to parse a futures spread, e.g. "CLX4-CLZ4"
-      public static FutureSpreadContract ParseFutureSpreadContract(string spreadString, Exchange exchange = Exchange.CME) {
+      public static FutureSpreadContract parse_future_spread_contract(string contract_txt, Exchange exchange = Exchange.CME) {
         // Split the string by '-' to handle each leg
-         var spreadParts = spreadString.Split('-');
+         var spreadParts = contract_txt.Split('-');
         if (spreadParts.Length < 2) {
            return null; // Not a valid spread format
         }

-         var spreadContract = new FutureSpreadContract { legs = new List<Tuple<double, FutureContract>>() };
+         var spreadContract = new FutureSpreadContract { legs = new List<Tuple<double, FutureTradableContract>>() };

         foreach (var part in spreadParts) {
            // Split each leg by '*', assuming the format is weight*future or just future
@@ -92,33 +175,32 @@ namespace bts.SilverPine {
            }

            // Parse the future contract
-            var futureContract = ParseFutureContract(futurePart, exchange);
+            var futureContract = parse_future_contract(futurePart, exchange);
            if (futureContract == null) {
               return null; // Failed to parse the future part
            }

            // Add the leg to the spread contract
-            spreadContract.legs.Add(new Tuple<double, FutureContract>(weight, futureContract));
+            spreadContract.legs.Add(new Tuple<double, FutureTradableContract>(weight, futureContract));
         }
-
         return spreadContract;
      }

-      public static ContractType DetermineFutureOrSpread(string contract) {
-         if (contract.Contains("-")) {
-            return ContractType.FutureSpread;
-         } else {
-            return ContractType.Future;
-         }
-      }
+      // public static ContractType DetermineFutureOrSpread(string contract) {
+      //    if (contract.Contains("-")) {
+      //       return ContractType.FutureSpread;
+      //    } else {
+      //       return ContractType.Future;
+      //    }
+      // }
      #endregion

      public static ContractType get_type(string contract_txt) {
-         FutureContract fc = ParseFutureContract(contract_txt);
+         FutureTradableContract fc = parse_future_contract(contract_txt);
         if (fc != null) {
            return ContractType.Future;
         }
-         FutureSpreadContract sc = ParseFutureSpreadContract(contract_txt);
+         FutureSpreadContract sc = parse_future_spread_contract(contract_txt);
         if (sc != null) {
            return ContractType.FutureSpread;
         }
@@ -133,11 +215,11 @@ namespace bts.SilverPine {
      }

      public static Tuple<ContractCategory, string> get_symbol_category(string symbol) {
-         ContractUtils.FutureContract fc = ParseFutureContract(symbol);
+         ContractUtils.FutureTradableContract fc = parse_future_contract(symbol);
         if (fc != null) {
            return Tuple.Create(ContractCategory.FutureString, fc.symbol);
         }
-         var sc = ParseFutureSpreadContract(symbol);
+         var sc = parse_future_spread_contract(symbol);
         if (sc != null) {
            return Tuple.Create(ContractCategory.FutureString, sc.symbol);
         }
@@ -165,18 +247,6 @@ namespace bts.SilverPine {
      public enum Exchange {
         CME
      }
-      public class FutureContract {
-         public string symbol;
-         public DateTime expiration;
-         public Exchange exchange;
-      }
-      public class FutureSpreadContract {
-         public List<Tuple<double, bts.SilverPine.ContractUtils.FutureContract>> legs;
-         public string symbol {
-            get {
-               return legs == null || legs.Count == 0 ? String.Empty : legs.ElementAt(0).Item2.symbol;
-            }
-         }
-      }
+
   }
 }
@@ -1,35 +1,105 @@
-namespace BCG {
+namespace bts.SilverPine {
   using BCG.Utilities.General;

-   using bts.api;
-   using bts.api.domain;
-   using bts.api.synthetic;
   using bts.md;
-   using bts.spark;
-   using bts.utils;
-
-   using mbts;
-   using mbts.co2;
-   using mbts.config;
-   using mbts.spark;
-   using mbts.ui;
-
-   using Spark.Core.Data;
-   using Spark.Core.Disposables;
+   using BCG;
   using Spark.Core.Reactive;

   using System;
-   using System.Collections.Concurrent;
   using System.Collections.Generic;
-   using System.Collections.ObjectModel;
   using System.Linq;
   using System.Reactive.Linq;
   using bts.api.core;
   using Spark.Api.MarketData;

   using SecurityKey = System.UInt32;
-   using System.Data;
-   using System.Reflection.Metadata;
+   using System.Data.Entity.Infrastructure;
+
+   public class FakeBBOStream : ViewStream, IWorker {
+      private string _symbol;
+      public BlueTable _table = new BlueTable(BBOStream._bbo_schema);
+      private DateTime _last_fired_time = DateTime.MinValue;
+      private TimeSpan _period = TimeSpan.FromMilliseconds(333);
+      private double _start_price;
+      private double _inc;
+      private int _total_count = 5;
+      private Random _rnd = new Random(DateTime.Now.Millisecond);
+      private string[] _months = new string[]{
+         "F","G", "H", "J","K", "M", "N","Q","U","V","X","Z"
+      };
+      private ViewStreamDatabase _vsd;
+      public FakeBBOStream(string contract_type, string exchange, string symbol, double start_price, double inc)
+      :
+      this(contract_type, exchange, symbol, start_price, inc, ViewStreamDatabase.instance) { }
+
+      public FakeBBOStream(string contract_type, string exchange, string symbol, double start_price, double inc, ViewStreamDatabase vsd) {
+         _symbol = symbol;
+         _start_price = start_price;
+         _inc = inc;
+         _vsd = vsd;
+         Log.assert(contract_type.ToLower() == "future");
+      }
+
+      protected override void myOnFirstSubscriber() {
+         _vsd.addWorker(this);
+      }
+
+      protected override void myOnLastSubscriber() {
+         _vsd.removeWorker(this);
+      }
+
+      private bool _random_truefase() {
+         int r = _rnd.Next(0, 100);
+         return r % 2 == 0;
+      }
+
+      private IBlueRow _perturb_row(IBlueRow er) {
+         MutableBlueRow rv = new MutableBlueRow(er);
+         double bid_price = rv.get<double>("bid_price");
+         bid_price += _inc;
+         double ask_price = bid_price + _inc;
+         rv.setValue("bid_price", bid_price);
+         rv.setValue("ask_price", ask_price);
+         return rv;
+      }
+
+      public void doWork() {
+         if (DateTime.Now - _last_fired_time < _period)
+            return;
+         _last_fired_time = DateTime.Now;
+         if (_table.Count == 0) {
+            // first fire_symbol
+            string year_code = DateTime.Now.Year.ToString().Last().ToString();
+            for (int i = 0; i < _total_count; i++) {
+               string month_code = _months[i % _months.Length];
+               string sec = $"{_symbol}{month_code}{year_code}";
+               double bid_price = _start_price + _rnd.Next(-10, 10) * _inc;
+               double ask_price = bid_price + _inc;
+               BlueRow nr = new BlueRow(_table.schema, _table.Count, _table.Count, sec,
+                                       bid_price, _rnd.Next(1, 10),
+                                       ask_price, _rnd.Next(1, 10));
+               _table.addRow(nr);
+               var dp = BlueDataUtility.build_dp(new BlueTable(_table), myLastValue as DataPacket);
+               myFireEvent(dp);
+            }
+         } else {
+            BlueTable deleted_rows = new BlueTable(_table.schema);
+            BlueTable added_rows = new BlueTable(_table.schema);
+            // go through all rows, select some at random, add them to deleted, get
+            // the replaced one and add it to added, updates table state and fire
+            for (int i = 0; i < _table.Count; i++) {
+               if (_random_truefase()) {
+                  IBlueRow er = _table[i];
+                  IBlueRow nr = _perturb_row(er);
+                  _table.deleteRow(er);
+                  _table.addRow(nr);
+               }
+            }
+            var dp = BlueDataUtility.build_dp_inc(new BlueTable(_table), myLastValue as DataPacket, true);
+            myFireEvent(dp);
+         }
+      }
+   }

   [DataStreamUserRespresentation("bbo")]
   public class BBOStream : ViewStream, IWorker {
@@ -39,7 +109,7 @@ namespace BCG {
      private int _id = 0;
      IReadOnlyDictionary<SecurityKey, BookTuple> _cached_state = null;
      MarketDataClient _md_client;
-      private readonly IBlueTableSchema _bbo_schema = BlueDataUtility.form_schema("bbo",
+      internal static IBlueTableSchema _bbo_schema = BlueDataUtility.form_schema("bbo",
                                                Tuple.Create("id", TypeCode.Int32),
                                                Tuple.Create("sec_key", TypeCode.UInt32),
                                                Tuple.Create("sec", TypeCode.String),
@@ -54,9 +124,11 @@ namespace BCG {
      }

      public BBOStream(ViewStreamDatabase vsd) {
-         bts.Log.Gen.info("BBOStream.cons");
-         _vsd = vsd;
-         _md_client = new MarketDataClient(new MDSource(TimeSpan.FromSeconds(0.5)));
+         // bts.Log.Gen.info("BBOStream.cons");
+         // _vsd = vsd;
+         // _md_client = new MarketDataClient(new MDSource(TimeSpan.FromSeconds(0.5)));
+         FakeBBOStream rs = new FakeBBOStream("Future", "CME", "ES", 5000, 0.25, vsd);
+         mySetRealSource(rs);
      }

      protected override void myOnFirstSubscriber() {
@@ -0,0 +1,241 @@
+namespace bts.SilverPine {
+   using BCG;
+
+   using System.Runtime.CompilerServices;
+
+   using ZstdSharp.Unsafe;
+
+   public class DateTimeRange {
+      public readonly DateTime range_beg;
+      public readonly DateTime range_end;
+      public DateTimeRange(DateTime range_beg_, DateTime range_end_) {
+         range_beg = range_beg_;
+         range_end = range_end_;
+      }
+      public static DateTimeRange parse(string range_beg_, string range_end_) {
+         DateTime range_beg = String.IsNullOrEmpty(range_beg_) ? DateTime.MinValue : DateTime.Parse(range_beg_);
+         DateTime range_end = String.IsNullOrEmpty(range_end_) ? DateTime.MaxValue : DateTime.Parse(range_end_);
+         return new DateTimeRange(range_beg, range_end);
+      }
+   }
+
+   public class FutureBarsLoader {
+      public static string get_db_table_name(BarSize bar_size, string symbol, bool is_continous) {
+         switch (bar_size) {
+            case BarSize.ONE_DAY: {
+                  if (is_continous) {
+                     return $"bars_future_{symbol}_continous_1day";
+                  } else
+                     return $"bars_future_{symbol}_all_1day";
+               }
+            case BarSize.ONE_MIN: {
+                  if (is_continous) {
+                     return $"bars_future_{symbol}_continous_1min";
+                  } else
+                     return $"bars_future_{symbol}_all_1min";
+               }
+         }
+         Log.assert(false);
+         return null;
+      }
+   }
+   public enum BarSize {
+      ONE_DAY = 0,
+      ONE_MIN
+   }
+
+   [DataStreamUserRespresentation("FuturesHistorical1DayBars")]
+   public class FuturesHistorical1DayBarsStream : ViewStream {
+
+      // this gets the raw bars for a symbol (perhaps make it into a raw bars stream?)
+      public FuturesHistorical1DayBarsStream(string symbol) :
+      this(symbol, DateTime.MinValue, DateTime.MaxValue) { }
+
+      // creates the non raw stream (ie for tradable contract, or continous contract)
+      public FuturesHistorical1DayBarsStream(string future_contract, DateTime range_beg, DateTime range_end)
+     : this(new ContractUtils.FutureContract(future_contract), new DateTimeRange(range_beg, range_end)) { }
+
+      // this creates a tradable contracts bars  streams or continous contracts stream. if get_all_rows is true, 
+      // it gets the raw table with all contracts for that symbol
+      public FuturesHistorical1DayBarsStream(ContractUtils.FutureContract future_contract,
+                                             DateTimeRange dt_range) {
+         DateTime beg = dt_range.range_beg;//.ToUniversalTime(),
+         DateTime end = dt_range.range_end;//.ToUniversalTime();
+         beg = beg.Date;
+         end = end.Date.Add(new TimeSpan(23, 59, 59));
+         // this returns rows from beg and end inclusive
+         Func<string, SQLiteStream> get_sqlite_str = (table_name) => {
+            string range_beg_iso8601 = beg.ToString("yyyy-MM-ddTHH:mm:ss"),
+               range_end_iso8601 = end.ToString("yyyy-MM-ddTHH:mm:ss");
+            string sql_str = $"select * from {table_name} where ts_event >= '{range_beg_iso8601}' AND ts_event <= '{range_end_iso8601}'";
+            return new SQLiteStream(SQLiteHelper.db_name, sql_str);
+         };
+         string symbol = future_contract.symbol();
+         if (future_contract.wildcard_contract != null) {
+            string table_name = FutureBarsLoader.get_db_table_name(BarSize.ONE_DAY, future_contract.symbol(), false);
+            ViewStream rs = get_sqlite_str(table_name);
+            //rs = new DateStandardizationStream(rs, "ts_event", DateConversion.ChopTime);
+            mySetRealSource(rs);
+            return;
+         }
+         if (future_contract.wildcard_contract != null) {
+            string table_name = FutureBarsLoader.get_db_table_name(BarSize.ONE_DAY, future_contract.symbol(), false);
+            ViewStream rs = get_sqlite_str(table_name);
+            //rs = new DateStandardizationStream(rs, "ts_event", DateConversion.ChopTime);
+            mySetRealSource(rs);
+            return;
+         }
+         if (future_contract.is_continous_contract) {
+            string table_name = FutureBarsLoader.get_db_table_name(BarSize.ONE_DAY, symbol, true);
+            ViewStream rs = get_sqlite_str(table_name);
+            //rs = new DateStandardizationStream(rs, "ts_event", DateConversion.ChopTime);
+            mySetRealSource(rs);
+            return;
+         } else {
+            string table_name = FutureBarsLoader.get_db_table_name(BarSize.ONE_DAY, future_contract.symbol(), false);
+            ViewStream rs = get_sqlite_str(table_name);
+            //rs = new DateStandardizationStream(rs, "ts_event", DateConversion.ChopTime);
+            //TODO: filter it 
+            rs = new FilterStream(rs, $"symbol = '{future_contract.contract_text}'");
+            mySetRealSource(rs);
+         }
+      }
+   }
+
+   [DataStreamUserRespresentation("FuturesHistorical1MinBars")]
+   public class FuturesHistorical1MinuteBarsStream : ViewStream {
+      // same as the above but for one min
+      // this gets the raw bars for a symbol
+      public FuturesHistorical1MinuteBarsStream(string symbol) :
+      this(symbol, DateTime.MinValue, DateTime.MaxValue) { }
+
+      // creates the non raw stream (ie for tradable contract, or continous contract)
+      public FuturesHistorical1MinuteBarsStream(string future_contract, DateTime range_beg, DateTime range_end)
+     : this(new ContractUtils.FutureContract(future_contract), new DateTimeRange(range_beg, range_end)) { }
+
+      // this creates a tradable contracts bars  streams or continous contracts stream. if get_all_rows is true, 
+      // it gets the raw table with all contracts for that symbol
+      public FuturesHistorical1MinuteBarsStream(ContractUtils.FutureContract future_contract,
+                                             DateTimeRange dt_range) {
+         Func<string, SQLiteStream> get_sqlite_str = (table_name) => {
+            string range_beg_iso8601 = dt_range.range_beg.ToString("yyyy-MM-ddTHH:mm:ss"),
+               range_end_iso8601 = dt_range.range_end.ToString("yyyy-MM-ddTHH:mm:ss");
+            //string sql_str = $"select * from {table_name} where DATE(ts_event) >= '{range_beg_iso8601}' AND DATE(ts_event) <= '{range_end_iso8601}'";
+            string sql_str = $"select * from {table_name} where ts_event >= '{range_beg_iso8601}' AND ts_event <= '{range_end_iso8601}'";
+            return new SQLiteStream(SQLiteHelper.db_name, sql_str);
+         };
+         string symbol = future_contract.symbol();
+         if (future_contract.wildcard_contract != null) {
+            string table_name = FutureBarsLoader.get_db_table_name(BarSize.ONE_MIN, future_contract.symbol(), false);
+            ViewStream rs = get_sqlite_str(table_name);
+            mySetRealSource(rs);
+            return;
+         }
+         if (future_contract.is_continous_contract) {
+            string table_name = FutureBarsLoader.get_db_table_name(BarSize.ONE_MIN, symbol, true);
+            ViewStream rs = get_sqlite_str(table_name);
+            mySetRealSource(rs);
+            return;
+         } else {
+            string table_name = FutureBarsLoader.get_db_table_name(BarSize.ONE_MIN, future_contract.symbol(), false);
+            ViewStream rs = get_sqlite_str(table_name);
+            // TODO: filter
+            rs = new FilterStream(rs, $"symbol = '{future_contract.contract_text}'");
+            mySetRealSource(rs);
+         }
+      }
+   }
+
+   [DataStreamUserRespresentation("FutureHistoricalBars")]
+   public class FutureHistoricalBarsStream : ViewStream {
+      public FutureHistoricalBarsStream(string fut_contract, string range_beg, string range_end) {
+         DateTime dt_range_beg = DateTime.Parse(range_beg);
+         DateTime dt_range_end = DateTime.Parse(range_end);
+         TimeSpan diff = dt_range_end - dt_range_beg;
+         BarSize bar_size = diff < TimeSpan.FromDays(1.0) ? BarSize.ONE_MIN : BarSize.ONE_DAY;
+         ViewStream rs = new FutureHistoricalBarsStream(fut_contract, bar_size.ToString(), dt_range_beg, dt_range_end);
+         mySetRealSource(rs);
+      }
+      public FutureHistoricalBarsStream(string fut_contract, string bar_size,
+                                        DateTime dt_range_beg, DateTime dt_range_end)
+                                        :
+      this(new ContractUtils.FutureContract(fut_contract), Enum.Parse<BarSize>(bar_size, true),
+         new DateTimeRange(dt_range_beg, dt_range_end)) { }
+
+      public FutureHistoricalBarsStream(ContractUtils.FutureContract fut_contract, BarSize bar_size,
+                                        DateTimeRange dt_range) {
+         switch (bar_size) {
+            case BarSize.ONE_DAY: {
+                  ViewStream rs = new FuturesHistorical1DayBarsStream(fut_contract, dt_range);
+                  mySetRealSource(rs);
+                  break;
+               }
+            case BarSize.ONE_MIN: {
+                  ViewStream rs = new FuturesHistorical1MinuteBarsStream(fut_contract, dt_range);
+                  mySetRealSource(rs);
+                  break;
+               }
+         }
+      }
+   }
+
+   [DataStreamUserRespresentation("FuturePriceDiff")]
+   public class FutureHistoricalBarsPriceDiffStream : ViewStream {
+
+      public FutureHistoricalBarsPriceDiffStream(string future_contract, string dt1, string dt2)
+      :
+      this(new ContractUtils.FutureContract(future_contract), DateTime.Parse(dt1), DateTime.Parse(dt2), "close", ViewStreamDatabase.instance) {
+      }
+
+
+      public FutureHistoricalBarsPriceDiffStream(string future_contract, DateTime dt1, DateTime dt2)
+      :
+      this(new ContractUtils.FutureContract(future_contract), dt1, dt2, "close", ViewStreamDatabase.instance) {
+      }
+
+      public FutureHistoricalBarsPriceDiffStream(string future_contract, DateTime dt1, DateTime dt2, string col)
+      :
+      this(new ContractUtils.FutureContract(future_contract), dt1, dt2, col, ViewStreamDatabase.instance) {
+      }
+
+
+      public FutureHistoricalBarsPriceDiffStream(ContractUtils.FutureContract fc, DateTime dt1, DateTime dt2, string col, ViewStreamDatabase vsd) {
+         BarSize bsize = dt2 - dt1 >= TimeSpan.FromDays(1.0) ? BarSize.ONE_DAY : BarSize.ONE_MIN;
+         ViewStream rs = new FutureHistoricalBarsStream(fc, BarSize.ONE_DAY, new DateTimeRange(dt1, dt2));
+
+         rs = new ColumnDifferenceStream(rs, r => {
+            DateTime r_dt = r.get_datetime("ts_event");
+            var rv = dt1.Equals(r_dt);
+            return rv;
+         },
+         //r => dt2.Equals(r.get_datetime("ts_event")), col, col);
+         r => {
+            DateTime r_dt = r.get_datetime("ts_event");
+            var rv = dt2.Equals(r_dt);
+            return rv;
+         }, col, col);
+         rs = new NewProjectStream(rs, "diff !");
+         mySetRealSource(rs);
+      }
+   }
+
+   // takes the contract name and expands it into multuiple columns corresponding to the contracts.
+   // the contract columns are a union, of all properties with some of them empty
+   public class ContractExpanderStream : ViewStream {
+      private static IBlueTableSchema map_schema(IBlueTableSchema schema) {
+         return schema.add_column(TypeCode.String, "contract_type");
+      }
+      public ContractExpanderStream(ViewStream parent_str, string contract_col_name) {
+         ViewStream rs = new MapStream(parent_str, (schema) => map_schema(schema),
+         r => {
+            string contract = r.get<string>(contract_col_name);
+            var ct = ContractUtils.get_type(contract);
+            MutableBlueRow rv = new MutableBlueRow(map_schema(r.schema), r);
+            rv.setValue("contract_type", ct.ToString());
+            return rv;
+         });
+         mySetRealSource(rs);
+      }
+   }
+
+}
@@ -0,0 +1,46 @@
+using BCG;
+
+namespace bts.SilverPine {
+   /*
+      ----------------------------------------------------------------------
+                                  *Expression*
+      ----------------------------------------------------------------------
+          //r=FutureHistoricalBars("ES*", "02/01/2024 09:30 AM", "02/01/2024 09:30 PM")
+          r=FutureHistoricalBars("ES*","One_Day", 02/01/2024, 02/01/2025)
+          r=contractExpander(r,"symbol")
+          r=filter(r,"contract_type='FutureSpread'")
+          r=aggregate(r,"symbol","volume:Sum","ts_event")
+          r=aggregate(r,"volume_sum:Max")
+          r=newProject(r,"volume_sum_max:volume symbol !")
+
+      ----------------------------------------------------------------------
+                                    *Schema*
+      ----------------------------------------------------------------------
+          symbol                          System.String
+          id                              System.Int32
+          volume                          System.Double
+   */
+
+   [DataStreamUserRespresentation("VolumeAgg")]
+   public class VolumeAggregationStream : ViewStream {
+      public VolumeAggregationStream(string future_contract, string contract_type, DateTime range_beg, DateTime range_end)
+      : this(future_contract, contract_type, range_beg, range_end, ViewStreamDatabase.instance) { }
+
+      public VolumeAggregationStream(string future_contract, string contract_type, string range_beg, string range_end)
+      :
+      this(future_contract, contract_type, DateTime.Parse(range_beg), DateTime.Parse(range_end), ViewStreamDatabase.instance) { }
+
+      public VolumeAggregationStream(string future_contract, string contract_type, DateTime range_beg, DateTime range_end, ViewStreamDatabase db) {
+         future_contract = future_contract.EndsWith("*") ? future_contract : future_contract + "*";
+         ViewStream r;
+         r = new FutureHistoricalBarsStream(future_contract,
+                                             range_beg.ToString(), range_end.ToString());
+         r = new ContractExpanderStream(r, "symbol");
+         r = new FilterStream(r, $"contract_type='{contract_type}'");
+         r = new NonIncGroupByStream(r, "symbol", "volume:Sum", "ts_event");
+         r = new NonIncGroupByStream(r, "volume_sum:Max");
+         r = new NewProjectStream(r, "volume_sum_max:volume symbol !");
+         mySetRealSource(r);
+      }
+   }
+}
@@ -1,68 +1,117 @@
+// namespace bts.SilverPine {
+//    using bts;
+//    using BCG;
+//    using MongoDB.Driver.Linq;
+//    using System.ComponentModel.DataAnnotations;
+//    using ZstdSharp.Unsafe;
+
+//    [DataStreamUserRespresentation("VWAP")]
+//    public class VWAPStream : ViewStream {
+//       // returns the VWAP stream for each contract 
+//       private static ViewStream build_vwap_stream(string symbol, DateTime begin_dt, DateTime end_dt) {
+//          var ohlcv_params = OHLCVStream.get_params(begin_dt, end_dt);
+//          ViewStream ohlc_str = new OHLCVStream(symbol, ohlcv_params);
+//          // stream is grouped per contract and then vwap is calculated per contract
+//          GlobalMapStream rs = new GlobalMapStream(ohlc_str, "ts_event", "contract",
+//          (rows, idx) => {
+//             Log.Gen.info($"VWAPStream.GlobalMapStream delegate n_rows: {rows.Count} idx: {idx}");
+//             IBlueRow ir = rows.ElementAt((int)idx);
+//             var sel_rows = rows.Skip((int)idx).Where(r => r.get_datetime("ts_event") >= end_dt && r.get_datetime("ts_event") <= begin_dt);
+//             var volumeprices = sel_rows.Select(r => Tuple.Create(r.get<int>("volume"), r.get<double>("close"))).ToList();
+//             double vwap = IndicatorCalc.calculate_VWAP(volumeprices);
+//             IBlueTableSchema schema = ir.schema.add_columns(Tuple.Create(typeof(double), "vwap"));
+//             MutableBlueRow nr = new MutableBlueRow(schema, ir);
+//             //vwap = -313.987;
+//             nr.setValue("vwap", vwap);
+//             return nr;
+//          });
+//          return rs;
+//       }
+//       // the args want dt1 > dt2, if its nto we, swap them
+//       static void swap_if_necessary(ref DateTime dt1, ref DateTime dt2) {
+//          if (dt1 < dt2) {
+//             DateTime aux = dt1;
+//             dt1 = dt2;
+//             dt2 = aux;
+//          }
+//       }
+
+//       // this assumes the ohlc_str has diff contracts
+//       public VWAPStream(string symbol_or_contract, DateTime begin_dt, DateTime end_dt) {
+//          swap_if_necessary(ref begin_dt, ref end_dt);
+//          Tuple<ContractUtils.ContractCategory, string> ccat_sym_contr = ContractUtils.get_symbol_category(symbol_or_contract);
+//          string symbol = String.Empty;
+//          symbol = ccat_sym_contr.Item2;
+//          var ohlcv_params = OHLCVStream.get_params(begin_dt, end_dt);
+//          ViewStream vwap_str = build_vwap_stream(symbol, begin_dt, end_dt);
+//          // vwap_str now has vwap for all contracts. if the arg was a symbol, we need to filter it for the active contract,
+//          // whihc we do with FutureActivesContractStream. If the arg was a contract we'll simply filter it
+//          ViewStream rs = vwap_str;
+//          switch (ccat_sym_contr.Item1) {
+//             case ContractUtils.ContractCategory.FutureString:
+//             case ContractUtils.ContractCategory.SpreadString:
+//                rs = new FilterStream(vwap_str, r => r.get<string>("contract").CompareTo(ccat_sym_contr.Item2) == 0);
+//                break;
+//             case ContractUtils.ContractCategory.Symbol:
+//                rs = new FuturesActiveContractStream(vwap_str);
+//                break;
+//          }
+//          rs = new FilterStream(rs, r => r.get_datetime("ts_event").CompareTo(begin_dt) == 0);
+//          rs = new NewProjectStream(rs, "! vwap");
+//          mySetRealSource(rs);
+//       }
+
+//       public VWAPStream(string symbol_or_contract, string begin_dt, string end_dt)
+//       :
+//       this(symbol_or_contract, DateTime.Parse(begin_dt), DateTime.Parse(end_dt)) { }
+//    }
+// }
+
+using System;
+
+using BCG;
+
+using bts.api;
+
 namespace bts.SilverPine {
-   using bts;
-   using BCG;
-   using MongoDB.Driver.Linq;
-   using System.ComponentModel.DataAnnotations;
-   using ZstdSharp.Unsafe;
+   /*
+      ----------------------------------------------------------------------
+                                  *Expression*
+      ----------------------------------------------------------------------
+          //nl("Find Settlement price of the SP500 future in the first week of 2023.")
+
+          r=FutureHistoricalBars("ES", "One_Day", "1/1/2024", "2/1/2024")
+          r=math(r, "vprice = volume * close")
+          r=aggregate(r, "vprice:Sum, volume:Sum")
+          r=math(r, "vwap = vprice_sum / volume_sum")
+          r=newProject(r, "vwap !")
+
+      ----------------------------------------------------------------------
+                                    *Schema*
+      ----------------------------------------------------------------------
+          id                              System.Int32
+          vwap                            System.Double
+   */

   [DataStreamUserRespresentation("VWAP")]
   public class VWAPStream : ViewStream {
-      // returns the VWAP stream for each contract 
-      private static ViewStream build_vwap_stream(string symbol, DateTime begin_dt, DateTime end_dt) {
-         var ohlcv_params = OHLCVStream.get_params(begin_dt, end_dt);
-         ViewStream ohlc_str = new OHLCVStream(symbol, ohlcv_params);
-         // stream is grouped per contract and then vwap is calculated per contract
-         GlobalMapStream rs = new GlobalMapStream(ohlc_str, "ts_event", "contract",
-         (rows, idx) => {
-            Log.Gen.info($"VWAPStream.GlobalMapStream delegate n_rows: {rows.Count} idx: {idx}");
-            IBlueRow ir = rows.ElementAt((int)idx);
-            var sel_rows = rows.Skip((int)idx).Where(r => r.get_datetime("ts_event") >= end_dt && r.get_datetime("ts_event") <= begin_dt);
-            var volumeprices = sel_rows.Select(r => Tuple.Create(r.get<int>("volume"), r.get<double>("close"))).ToList();
-            double vwap = IndicatorCalc.calculate_VWAP(volumeprices);
-            IBlueTableSchema schema = ir.schema.add_columns(Tuple.Create(typeof(double), "vwap"));
-            MutableBlueRow nr = new MutableBlueRow(schema, ir);
-            //vwap = -313.987;
-            nr.setValue("vwap", vwap);
-            return nr;
-         });
-         return rs;
-      }
-      // the args want dt1 > dt2, if its nto we, swap them
-      static void swap_if_necessary(ref DateTime dt1, ref DateTime dt2) {
-         if (dt1 < dt2) {
-            DateTime aux = dt1;
-            dt1 = dt2;
-            dt2 = aux;
-         }
-      }
-
-      // this assumes the ohlc_str has diff contracts
-      public VWAPStream(string symbol_or_contract, DateTime begin_dt, DateTime end_dt) {
-         swap_if_necessary(ref begin_dt, ref end_dt);
-         Tuple<ContractUtils.ContractCategory, string> ccat_sym_contr = ContractUtils.get_symbol_category(symbol_or_contract);
-         string symbol = String.Empty;
-         symbol = ccat_sym_contr.Item2;
-         var ohlcv_params = OHLCVStream.get_params(begin_dt, end_dt);
-         ViewStream vwap_str = build_vwap_stream(symbol, begin_dt, end_dt);
-         // vwap_str now has vwap for all contracts. if the arg was a symbol, we need to filter it for the active contract,
-         // whihc we do with FutureActivesContractStream. If the arg was a contract we'll simply filter it
-         ViewStream rs = vwap_str;
-         switch (ccat_sym_contr.Item1) {
-            case ContractUtils.ContractCategory.FutureString:
-            case ContractUtils.ContractCategory.SpreadString:
-               rs = new FilterStream(vwap_str, r => r.get<string>("contract").CompareTo(ccat_sym_contr.Item2) == 0);
-               break;
-            case ContractUtils.ContractCategory.Symbol:
-               rs = new FuturesActiveContractStream(vwap_str);
-               break;
-         }
-         rs = new FilterStream(rs, r => r.get_datetime("ts_event").CompareTo(begin_dt) == 0);
-         rs = new NewProjectStream(rs, "! vwap");
-         mySetRealSource(rs);
-      }
-
-      public VWAPStream(string symbol_or_contract, string begin_dt, string end_dt)
+      public VWAPStream(string symbol_or_contract, DateTime beg_dt, DateTime end_dt)
      :
-      this(symbol_or_contract, DateTime.Parse(begin_dt), DateTime.Parse(end_dt)) { }
+      this(symbol_or_contract, beg_dt, end_dt, ViewStreamDatabase.instance) { }
+
+      public VWAPStream(string symbol_or_contract, DateTime beg_dt, DateTime end_dt, ViewStreamDatabase db) {
+         var fc = new ContractUtils.FutureContract(symbol_or_contract);
+         ViewStream r = new FutureHistoricalBarsStream(fc, BarSize.ONE_DAY, new DateTimeRange(beg_dt, end_dt));
+
+         r = new MathStream(r,
+                 "vprice = volume * close");
+         r = new NonIncGroupByStream(r,
+                                      "vprice:Sum, volume:Sum");
+         r = new MathStream(r,
+                             "vwap = vprice_sum / volume_sum");
+         r = new NewProjectStream(r,
+                                   "vwap !");
+         mySetRealSource(r);
+      }
   }
-}
+}
@@ -75,4 +75,36 @@ namespace bts.SilverPine {
         mySetRealSource(rs);
      }
   }
+}
+
+
+namespace bts.SilverPine {
+   using BCG;
+
+   using bts.api;
+
+   using Newtonsoft.Json.Serialization;
+
+   [DataStreamUserRespresentation("FuturesContinousContract")]
+   public class FuturesContinousContractStream : ViewStream {
+      // the argument is symbol, like "ES" etc and not the contract itself
+      public FuturesContinousContractStream(string symbol) : this(symbol, ViewStreamDatabase.instance) { }
+      public FuturesContinousContractStream(string symbol, ViewStreamDatabase db) {
+         ViewStream r = new FuturesHistorical1DayBarsStream(symbol);
+         // filtering for futures only
+         r = new FilterStream(r, r => {
+            string contract = r.get<string>("symbol");
+            var fc = new ContractUtils.FutureContract(contract);
+            return fc.future_tradable_contract != null;
+         });
+         r = new NonIncGroupByStream(r,
+                                      "ts_event",
+                                      "volume:Max");
+         r = new NewProjectStream(r,
+                                   "ts_event symbol !");
+         r = new SortStream(r,
+                             "-ts_event");
+         mySetRealSource(r);
+      }
+   }
 }
@@ -2,6 +2,93 @@ namespace bts.SilverPine {
   using bts;
   using BCG;
   using MongoDB.Driver;
+
+   // OHLCV request breakdown
+   // symbol_or_contract (if symbol, get continous, otherwise get raw and filter it)
+   // bar_size (one min, one day)
+   // live/historic
+   // time range
+
+   // internal code oranization only
+   public class OHLCV_1min_stream : ViewStream {
+
+      public OHLCV_1min_stream(string symbol_or_contract, string range_beg, string range_end) :
+      this(symbol_or_contract, DateTime.Parse(range_beg), DateTime.Parse(range_end)) { }
+
+      public OHLCV_1min_stream(string symbol_or_contract, DateTime range_beg, DateTime range_end) {
+         Func<string, bool, SQLiteStream> get_sqlite_str = (symbol, is_continous_contracts) => {
+            string table_name = is_continous_contracts ? $"ohlcv_1min_{symbol}_continous" : $"{symbol}_ohlcv_1min";
+            string db_str = $"Data Source={SQLiteHelper.location}{SQLiteHelper.db_name};Version=3";
+            string range_beg_iso8601 = range_beg.ToString("yyyy-MM-ddTHH:mm:ss"),
+               range_end_iso8601 = range_end.ToString("yyyy-MM-ddTHH:mm:ss");
+            string sql_str = $"select * from {table_name} where DATE(ts_event) >= '{range_beg_iso8601}' AND DATE(ts_event) <= '{range_end_iso8601}'";
+            return new SQLiteStream(db_str, sql_str);
+         };
+
+         Tuple<ContractUtils.ContractCategory, string> tup = ContractUtils.get_symbol_category(symbol_or_contract);
+         var contract_cat = tup.Item1;
+         switch (contract_cat) {
+            case ContractUtils.ContractCategory.Symbol: {
+                  ViewStream res = get_sqlite_str(tup.Item2, true);
+                  mySetRealSource(res);
+                  // this creates the continous stream for 1 min bars, but we store it in the DB, so we just read it from it
+                  // ViewStream ohlcv_1min = get_sqlite_str(tup.Item2, false);
+                  // // the following is whats written to the DB in the _contionous table
+                  // ohlcv_1min = new DateTimeExpanstionStream(ohlcv_1min, "ts_event");
+                  // ViewStream active_contr_str = new FuturesActiveContractStream(tup.Item2);
+                  // ViewStream res = new NonIncJoinStream(ohlcv_1min, "ts_event_date symbol", "ts_event ts_event_date symbol open high low close volume",
+                  //                                     active_contr_str, "ts_event contract", "ts_event:date contract:continous_contract");
+                  // res = new NullFilterStream(res, "continous_contract", true);
+                  // res = new NewProjectStream(res, "ts_event date continous_contract open high low close volume !");
+                  // mySetRealSource(res);
+                  break;
+               }
+            default: {
+                  ViewStream sql_str = get_sqlite_str(tup.Item2, false);
+                  ViewStream rs = new FilterStream(sql_str, $"symbol='{symbol_or_contract}'");
+                  // filter here
+                  mySetRealSource(rs);
+                  break;
+               }
+         }
+      }
+   }
+
+   // this is to make bigger bars form smaller bars. Right now the assumption it only works
+   // with symbols vs contracts, and the 1min <= aggregation_period < 1day
+   [DataStreamUserRespresentation("OHLCVAggregated")]
+   public class OHLCVAggregatedStream : ViewStream {
+      public OHLCVAggregatedStream(string symbol, string agregation_period, string range_beg, string range_end) :
+      this(symbol, TimeBasedFilter.parse_duration(agregation_period), DateTime.Parse(range_beg), DateTime.Parse(range_end)) {
+      }
+
+      public OHLCVAggregatedStream(string symbol, TimeSpan aggregation_period, DateTime range_beg, DateTime range_end) {
+         Tuple<ContractUtils.ContractCategory, string> tup = ContractUtils.get_symbol_category(symbol);
+         Log.assert(tup.Item1 == ContractUtils.ContractCategory.Symbol);
+         if (aggregation_period < TimeSpan.FromDays(1.0) &&
+            aggregation_period > TimeSpan.FromMinutes(1.0)) {
+            ViewStream r = new OHLCV_1min_stream(symbol, range_beg, range_end);
+            r = new DateTimeExpanstionStream(r,
+                                              "ts_event");
+            double tot_secs = aggregation_period.TotalSeconds;
+            r = new MathStream(r,
+                                $"bucket = Floor(ts_event_seconds_of_day / {tot_secs})");
+            r = new NonIncGroupByStream(r,
+                                         "bucket",
+                                         // column:operation
+                                         "open:first, high:max, low:min, close:last, volume:sum",
+                                         "ts_event");
+            r = new SortStream(r,
+                                "-ts_event");
+            mySetRealSource(r);
+         } else {
+            // fix it later
+            Log.assert(false);
+         }
+      }
+   }
+
+
   [DataStreamUserRespresentation("OHLCV")]
   // This stream retreives the OHLCV table as is, and fires it as-is. 
   // The only logic here is as follows:
@@ -80,12 +167,13 @@ namespace bts.SilverPine {
         switch (ohlcv_params.type) {
            case OHLCVType.LIVE:
            case OHLCVType.LIVE_AND_HISTORIC:
+               // TODO
               Log.assert(false);
               break;
            case OHLCVType.HISTORIC:
               // TODO: assuming 1 day only for historic               
               string table_name = get_table_from_barsize(ohlcv_params.bar_size, symbol);// $"{symbol}_OHLCV_1DAY";
-               //string table_name = $"{symbol}_{ohlcv_params.bar_size}";
+                                                                                         //string table_name = $"{symbol}_{ohlcv_params.bar_size}";
               string db_str = $"Data Source={_location}{_db_name};Version=3";
               string sql_str = $"select * from {table_name}";
               // "Data Source=C:\\Users\\m_tah\\development\\data\\stream_data.db;Version=3;"
@@ -111,6 +199,11 @@ namespace bts.SilverPine {
      :
      this(symbol, Enum.Parse<OHLCVBarSize>(bar_size)) { }

+      // aggregates the bars
+      public OHLCVStream(string symbol, TimeSpan agregation_period) {
+
+      }
+


   }
@@ -3,49 +3,22 @@ namespace bts.SilverPine {

   using bts;

-   using DnsClient.Protocol;
-
-   using MongoDB.Driver;
-
   [DataStreamUserRespresentation("Settlement")]
   public class SettlementPriceStream : ViewStream {

      public SettlementPriceStream(string future_name, string date_txt)
-      : this(future_name, date_txt, String.Empty) { }
+      : this(future_name, date_txt, date_txt) { }

      public SettlementPriceStream(string future_name, string begin_dt, string end_dt)
      : this(future_name, DateTime.Parse(begin_dt).Date, DateTime.Parse(end_dt).Date) { }

      public SettlementPriceStream(string symbol_or_contract, DateTime begin_dt)
-      : this(symbol_or_contract, begin_dt, DateTime.MinValue) { }
+      : this(symbol_or_contract, begin_dt, begin_dt) { }

      public SettlementPriceStream(string symbol_or_contract, DateTime begin_dt, DateTime end_dt) {
-         // DateTime begin_dt = DateTime.Parse(begin_dt_txt);
-         // DateTime end_dt = String.IsNullOrEmpty(end_dt_txt) ? DateTime.MinValue : DateTime.Parse(end_dt_txt);
-         Tuple<ContractUtils.ContractCategory, string> ccat_sym_contr = ContractUtils.get_symbol_category(symbol_or_contract);
-         ContractUtils.ContractCategory contr_cat = ccat_sym_contr.Item1;
-         string symbol = ccat_sym_contr.Item2;
-         ViewStream settl_str = new OHLCVStream(symbol);
-         // TODO: close != settlement, get settlement from somewhere
-         ViewStream rs = new NewProjectStream(settl_str, "close:settlement");
-         switch (contr_cat) {
-            case ContractUtils.ContractCategory.FutureString:
-            case ContractUtils.ContractCategory.SpreadString:
-               rs = new FilterStream(settl_str, r => r.get<string>("contract").CompareTo(symbol_or_contract) == 0);
-               break;
-            case ContractUtils.ContractCategory.Symbol:
-               rs = new FuturesActiveContractStream(settl_str);
-               break;
-         }
-         if (DateTime.MinValue.Equals(end_dt)) {
-            rs = new FilterStream(rs, r => r.get_datetime("ts_event").CompareTo(begin_dt) == 0);
-         }
-         rs = new FilterStream(rs, r => {
-            DateTime dt = r.get_datetime("ts_event");
-            return dt <= begin_dt && dt >= end_dt;
-         });
-         rs = new MathStream(rs, "settlement = close * 1.0");
-         rs = new NewProjectStream(rs, " ! ts_event:date settlement");
+         ContractUtils.FutureContract fc = new ContractUtils.FutureContract(symbol_or_contract);
+         ViewStream rs = new FutureHistoricalBarsStream(fc, BarSize.ONE_DAY, new DateTimeRange(begin_dt, end_dt));
+         rs = new NewProjectStream(rs, "symbol ts_event:date close:settlement !");
         mySetRealSource(rs);
      }

@@ -41,70 +41,74 @@
      },
      {
         "category": "VWAP",
-         "description": "This is a few shot learning example for an NL to DSL. The DSL function in question is VWAP (symbol_or_contract, datetime1, datetime2). The first argument is a future (like CL, for crude oil, or ES for SP 500) or a specific future contract like crude oil Dec 24 future (CLZ4) or SP 500 Dec 2024 (ESZ4). The second argument is where the VWAP has to be calculated (also the beginning of the time window), and the third argument is datetime specifying the end of the time window. So, for example, a 20-day VWAP on 9/30/2023, the first argument will be 9/30/2023, and the third argument will be 9/10/2023 (20 days prior). This will also have datetimes expressed as Today, Yesterday, Last Friday, Last Sunday, etc. For the purpose of this few-shot prompts, let's assume today is 10/11/2024.",
+         "description": "This is a few shot learning example for an NL to DSL. The DSL function in question is VWAP (symbol_or_contract, perio_end_date, period_start_date). The first argument is a future (like CL, for crude oil, or ES for SP 500) or a specific future contract like crude oil Dec 24 future (CLZ4) or SP 500 Dec 2024 (ESZ4). The second and third argument define the end and start of the VWAP period, respectively. So for example, if we are calculating the 10 day VWAP on 10/15/2023, the period will be from 10/15/2023 going back to 10/05/2023. so the second argument will 10/05/2023 and the third argument will be 10/15/2023. The NL prompt  will also have datetimes expressed as Today, Yesterday, Last Friday, Last Sunday, First day of quarter, Today etc. You will need to translate these as dates that need to be passed to the VWAP DSL function. ",
         "prompt": "What was the VWAP for SP 500 Future over the past 30 days?",
         "examples": [
            {
               "natural_language": "Find the 20 day VWAP of the SP future on 04/22/2023.",
-               "dsl": "VWAP(\"ES\", 4/22/2023, 4/02/2023)"
+               "dsl": "VWAP(\"ES\", 4/02/2023, 4/22/2023)"
            },
            {
               "natural_language": "Find Yesterday's 10 day VWAP of the Crude Oil Future.",
-               "dsl": "VWAP(\"CL\", 10/10/2024, 10/1/2024)"
+               "dsl": "VWAP(\"CL\", 10/01/2024, 10/10/2024)"
            },
            {
               "natural_language": "Find today's 30-day VWAP of the 10 year treasury future.",
-               "dsl": "VWAP(\"ZN\", 10/11/2024, 9/11/2024)"
+               "dsl": "VWAP(\"ZN\", 9/11/2024, 10/11/2024)"
            },
            {
               "natural_language": "Find the 10 hour VWAP of the Natural Gas future from yesterday at 5:00 PM.",
-               "dsl": "VWAP(\"NG\", 9/27/2024 17:00:00, 9/27/2024 7:00:00)"
+               "dsl": "VWAP(\"NG\", 9/27/2024 7:00:00, 9/27/2024 17:00:00)"
            }
         ]
      },
      {
-         "category": "Price Diff",
-         "description": "This is a few shot learning example for an NL to DSL. The DSL function in question is OHLCVPriceDiff (symbol_or_contract, datetime1, datetime2). The first argument is a future (like CL, for crude oil) or a specific future contract like crude oil Dec 24 future (CLZ4). The second and third arguments are dates, that the function finds the difference of prices at. Assume today is 10/15/2024",
-         "prompt": "What is the difference in  SP 500 Future from July 10th to July 14th?",
+         "category": "FuturePriceDiff",
+         "description": "This is a few-shot learning example for an NL to DSL. The DSL function in question is FuturePriceDiff(symbol_or_contract, datetime1, datetime2). The first argument is a future (like CL, for crude oil, or ES for SP 500 and so on and so forth) or a specific future contract like crude oil Dec 24 future (CLZ4). The second and third arguments are datetimes (usually dates) that the function finds the difference of prices at. If the dates are given like \"first day of quarter\", \"first week of August\" I expect you to convert them into exact dates since the function only takes specific dates. The second argument should be the smaller datetime and the 3rd argument should be the end of the range, ie the higher one. Any prompt asking for price difference or price move should call FuturePriceDiff. If the dates you compute to pass to FuturePriceDiff are dates on which futures exchange might be closed (like 1/1, 12/25, Thanksgiving etc)  please adjust the date to the nearest viable trading date, for example if the date falls on christmas day, change it to 12/24 and so on and so on and so forth, or the date happens to be a saturday or sunday, change it to Friday or Monday depending on which is closest",
+         "prompt": "What is the difference in SP 500 Future from July 10th to July 14th?",
         "examples": [
            {
               "natural_language": "What is the difference in ZN price from July 10th to July 14th?",
-               "dsl": "OHLCVPriceDiff(\"ZN\", 7/14/2024, 7/10/2024)"
+               "dsl": "FuturePriceDiff(\"ZN\", 7/10/2024, 7/14/2024)"
            },
            {
               "natural_language": "What is the price difference in SP Future price from Sept 10th and Sept 5th?",
-               "dsl": "OHLCVPriceDiff(\"ES\", 9/10/2024, 9/5/2024)"
+               "dsl": "FuturePriceDiff(\"ES\", 9/5/2024, 9/10/2024)"
            },
            {
               "natural_language": "What is the price difference in Natural Gas Dec Future price from Aug 30th and Aug 20th?",
-               "dsl": "OHLCVPriceDiff(\"NGZ4\", 8/30/2024, 8/20/2024)"
+               "dsl": "FuturePriceDiff(\"NGZ4\", 8/20/2024, 8/30/2024)"
+            },
+            {
+               "natural_language": "What is the price move in  SP 500 Future from Aug 30th 2023 to October 10th 2023",
+               "dsl": "FuturePriceDiff(\"ES\", 8/30/2023, 10/10/2023)"
+            },
+            {
+               "natural_language": "What is the price move in  SP 500 Future during teh first quarter of last year",
+               "dsl": "FuturePriceDiff(\"ES\", 1/1/2023, 3/31/2023)"
            }
         ]
      },
      {
-         "category": "Absolute Volume",
-         "description": "this is a few shot learning example for an NL to DSL. The DSL function in question is AbsVolume(symbol, ContractType, datetime1, datetime2), which computes the highest volume in the date range, specified by datetime1, and datetime2. The first argument is a future symbol (like CL for crude oil, ES for SP 500 etc).  The second argument is the ContractType, which is \"Future\" for futures and \"FutureSpread\" for future spreads. the 3rd argument is the begin date and the 4th argument is the end date, between which the max volume is computed. If no future is specified use \"*\" from the first arugment. For this prompt, assume today is 10/11/2024 ",
+         "category": "Aggregate Volume",
+         "description": "this is a few shot learning example for an NL to DSL. The DSL function in question is VolumeAgg(symbol, contract_type, datetime1, datetime2), which computes the highest volume in the date range, specified by datetime1, and datetime2. The first argument is a future symbol (like CL for crude oil, ES for SP 500 etc).  The second argument is the ContractType, which is \"Future\" for futures and \"FutureSpread\" for future spreads. the 3rd argument is the begin date and the 4th argument is the end date, between which the max volume is computed. If no future is specified use \"*\" from the first arugment.",
         "prompt": "Which  SP 500 Future traded the most between 7/14/2024 and 7/10/2024",
         "examples": [
            {
-               "natural_language": "Which SP Future traded the most between 7/14/2024 and 7/10/2024",
-               "dsl": "AbsVolume(\"ES\", \"Future\", 7/14/2024, 7/10/2024)"
+               "natural_language": "Which SP Future traded the most between 7/10/2024 and 7/14/2024",
+               "dsl": "VolumeAgg(\"ES\", \"Future\", 1/1/2024, 1/7/2024)"
            },
            {
-               "natural_language": "Which Gold Spread traded the most between Today and Sept 20th",
-               "dsl": "AbsVolume(\"GC\", \"FutureSpread\", 10/1/2024, 9/20/2024)"
+               "natural_language": "Which Gold Spread traded the most in the first month of this year",
+               "dsl": "VolumeAgg(\"GC\", \"FutureSpread\", 1/1/2024, 1/31/2024)"
            },
            {
-               "natural_language": "Which 10 year Spread traded the highest volume last week?",
-               "dsl": "AbsVolume(\"ZN\", \"FutureSpread\", 9/27/2024, 9/23/2024)"
+               "natural_language": "Which 10 year Spread traded the highest volume last week to now?(in this example(only) we are assuming today is 11/26/2024)",
+               "dsl": "VolumeAgg(\"ZN\", \"FutureSpread\", 11/19/2024, 11/26/2024)"
            },
            {
-               "natural_language": "Which Corn Spread traded the most today?",
-               "dsl": "AbsVolume(\"ZC\", \"FutureSpread\", 10/1/2024, 10/1/2024)"
-            },
-            {
-               "natural_language": "Which Gold Future traded the most today?",
-               "dsl": "AbsVolume(\"GC\", \"Future\", 9/30/2024, 9/30/2024)"
+               "natural_language": "Which Corn Spread traded the most today (in this example(only) we are assuming today is 11/26/2024)?",
+               "dsl": "VolumeAgg(\"ZC\", \"FutureSpread\", 11/26/2024, 11/26/2024)"
            }
         ]
      },
@@ -218,6 +222,17 @@
               "dsl": "r=standardDeviation(\"ZN\", \"30d\")\nr=math(r, \"abs_daily_change = Abs(close - open) comp = std_dev * 2.0\")\nr=filter(r,\"abs_daily_change >= comp\")"
            }
         ]
+      },
+      {
+         "category": "Price Movement",
+         "description": "This is a few shot learning example for an NL to DSL. In the NL prompt the user is asking for the average price movement for e-minis (aka ES aka SP Futures aka SP 500 Futures)) on Wednesday between 9:30 am and 9:40 AM?  the multiline DSL is as follows (along with the explaination in comments before each line, dont include the comments in the asnwer):  //following line gets the 1 min ohlcv bars of ES \n r=OHLCV_1min_stream(\"ES\")\n //this passes it to the filter stream, which filters the rows for Wednesday, and time bw 9:30 and 9:40.\nr=filter(r,\"ts_event_day = 'Wednesday' and ts_event_hour = 9 and ts_event_min >= 30 and ts_event_min < 40\")\n// this aggregates it using the date as the agregation key, it find the max of the column called high, and the low of the column called min, to find the max and the min prices in that time window\nr=aggregate(r, \"ts_event_date\", \"high: max, low: min\")\n/// this then passes the previous result to the math stream operator which calculates the move, which is the difference between high_max and low_min (columns that were added from the previous aggregate)\nr=math(r,\"move = high_max - low_min\")\n// At the end we compute the mean of the move column, notice this is the total mean, so we are not providing an aggregation col to agregate\nr=aggregate(r, \"move:mean\")\n// we project out every col by the ! operator and call rename move_mean as average_vol\nr=newProject(r,\"move_mean:average_move !\")",
+         "prompt": "What is the average price movement for SP Futures (ES) on Wednesday between 9:30 am and 9:40 AM ?",
+         "examples": [
+            {
+               "natural_language": "What is the average price movement for SP Futures (ES) on Wednesday between 9:30 am and 9:40 AM?",
+               "dsl": "r=OHLCV_1min_stream(\"ES\")\nr=filter(r,\"ts_event_day = 'Wednesday' and ts_event_hour = 9 and ts_event_min >= 30 and ts_event_min < 40\")\nr=aggregate(r, \"ts_event_date\", \"high: max, low: min\")\nr=math(r,\"move = high_max - low_min\")\nr=aggregate(r, \"move:mean\")\nr=newProject(r,\"move_mean:average_move !\")"
+            }
+         ]
      }
   ]
 }
@@ -1,4 +1,4 @@
-namespace sbts.SilverPine {
+namespace bts.SilverPine {
   using BCG;

   using bts;
@@ -1,56 +1,42 @@
-cmake_minimum_required(VERSION 3.5)
+cmake_minimum_required(VERSION 3.10)
 project(Spark.Admin.WebApp)

+include(GNUInstallDirs)
 include(${CMAKE_SOURCE_DIR}/cmake/safe_dep_glob.cmake)
-include(${CMAKE_SOURCE_DIR}/cmake/Protobuf.cmake)
+include(${CMAKE_SOURCE_DIR}/cmake/SymLink.cmake)
+include(${CMAKE_SOURCE_DIR}/cmake/ProtobufGenerateJs.cmake)
+include(${CMAKE_SOURCE_DIR}/cmake/ProtobufGenerateFs.cmake)

-# dependencies
-set(PROTO_GLOBS
+file(
+  GLOB_RECURSE ${PROJECT_NAME}_PROTO_SOURCES
+  RELATIVE ${CMAKE_CURRENT_SOURCE_DIR}
  ../lib/api/*.proto
  ../lib/api/services/*.proto
  ../lib/domain_model/*.proto
  ../lib/sparkproto/*.proto
  ../lib/md/protos/*.proto
  ../../externals/linux/include/google/protobuf/descriptor.proto
-  ../../externals/linux/include/google/protobuf/timestamp.proto)
+  ../../externals/linux/include/google/protobuf/timestamp.proto
+)

-file(
-  GLOB_RECURSE _proto_rel_SOURCES
-  RELATIVE ${CMAKE_CURRENT_SOURCE_DIR}
-  ${PROTO_GLOBS})
-
-file(
-  GLOB_RECURSE _proto_abs_SOURCES
-  ${PROTO_GLOBS})
-
-file(
-  GLOB_RECURSE _ts_SOURCES
-  RELATIVE ${CMAKE_CURRENT_SOURCE_DIR}
-  src/*.ts)
-
-file(
-  GLOB_RECURSE _fs_SOURCES
-  RELATIVE ${CMAKE_CURRENT_SOURCE_DIR}
-  src/*.fs)
+add_custom_command(
+  OUTPUT ${CMAKE_CURRENT_SOURCE_DIR}/package-lock.json
+  COMMAND npm install --include=dev
+  DEPENDS ${CMAKE_CURRENT_SOURCE_DIR}/package.json
+  WORKING_DIRECTORY ${CMAKE_CURRENT_SOURCE_DIR}
+)

 set(_tools ${CMAKE_SOURCE_DIR}/tools)
 set(_quiet_exec ${_tools}/quiet_exec.sh)
 set(_venv_exec ${_tools}/venv_exec.sh)
 set(_tools_py ${CMAKE_SOURCE_DIR}/tools/py)

-# The tsconfig file is generated as a function of the cmake build type, and
-# binary dir. Depending on the build type, it extends either
-# 'tsconfig.debug.json' or 'tsconfig.release.json'.
 set(_tsconfig_base "${CMAKE_CURRENT_SOURCE_DIR}/tsconfig.$<IF:$<CONFIG:Debug>,debug,release>.json")
-set(WEBPACK_MODE $<IF:$<CONFIG:Debug>,"development","production">)
 set(_tsconfig "${CMAKE_CURRENT_BINARY_DIR}/tsconfig.json")

 set(_venv_python ${CMAKE_SOURCE_DIR}/.venv/bin/python)
 set(_gen_tsconfig_py ${_tools_py}/gen_tsconfig.py)
 set(_build_admin_web_app_py ${_tools_py}/build_admin_web_app.py)
-
-set(_gen_tsconfig_deps ${_venv_exec} ${_venv_python} ${_gen_tsconfig_py} ${_tsconfig_base})
-
 set(_build_admin_web_app_deps
  ${_quiet_exec}
  ${_venv_exec}
@@ -60,49 +46,142 @@ set(_build_admin_web_app_deps
  ${CMAKE_CURRENT_SOURCE_DIR}/webpack.config.js
  ${CMAKE_SOURCE_DIR}/src/lib/api/service_compiler/templates/fs/fable.fs
  ${CMAKE_CURRENT_SOURCE_DIR}/package-lock.json
-  ${_proto_rel_SOURCES}
+  ${${PROJECT_NAME}_PROTO_SOURCES}
  ${_api_ts_SOURCES}
  ${_ui_fs_SOURCES}
  ${_tsconfig})

 update_deps_file("${_build_admin_web_app_deps}")

-function(npm_install path)
-  add_custom_command(
-    OUTPUT ${path}/package-lock.json
-    COMMAND "npm" ARGS "install" "--include=dev"
-    DEPENDS ${path}/package.json
-    WORKING_DIRECTORY ${path}
-    COMMENT "Run 'npm install' in ${path}")
-endfunction()
-
-npm_install(${CMAKE_CURRENT_SOURCE_DIR})
-
 add_custom_command(
  OUTPUT ${_tsconfig}
  COMMAND ${_venv_exec}
  ARGS
-  ${_venv_python}
-  ${_gen_tsconfig_py}
-  --config-base ${_tsconfig_base}
-  --config-out ${_tsconfig}
-  --build-out ${CMAKE_CURRENT_BINARY_DIR}
-  DEPENDS ${_gen_tsconfig_deps})
+    ${_venv_python}
+    ${_gen_tsconfig_py}
+    --config-base ${_tsconfig_base}
+    --config-out ${_tsconfig}
+    --build-out ${CMAKE_CURRENT_BINARY_DIR}
+  DEPENDS
+    ${_venv_exec}
+    ${_venv_python}
+    ${_gen_tsconfig_py}
+    ${_tsconfig_base}
+    ${CMAKE_CURRENT_SOURCE_DIR}/package-lock.json
+)
+
+protobuf_generate_js(
+  TARGET_NAME generate_js_protos
+  PROTO_INCLUDE_DIRS
+    ..
+    ../../externals/linux/include
+  PROTO_SOURCES ${${PROJECT_NAME}_PROTO_SOURCES}
+  OUTPUT ${CMAKE_CURRENT_BINARY_DIR}/proto.js
+)
+
+protobuf_generate_fs(
+  TARGET_NAME generate_fs_protos
+  PROTO_INCLUDE_DIRS
+    ..
+    ../../externals/linux/include
+  PROTO_SOURCES ${${PROJECT_NAME}_PROTO_SOURCES}
+  OUT_DIR ${CMAKE_CURRENT_BINARY_DIR}/fable
+)
+
+# Each input file maps to multiple output files with different extensions
+# (js, js.map, d.ts, fs). Transpile ts -> d.ts (and js), then d.ts -> fs.
+# Who knew front-end development could be so complicated? `\_(=_=)_/`
+file(
+  GLOB_RECURSE ${PROJECT_NAME}_TS_SOURCES
+  RELATIVE ${CMAKE_CURRENT_SOURCE_DIR}
+  src/*.ts
+)
+set(${PROJECT_NAME}_JS_PROTOS)
+set(${PROJECT_NAME}_TS_PROTOS)
+set(${PROJECT_NAME}_FS_PROTOS)
+foreach(_ts_source ${${PROJECT_NAME}_TS_SOURCES})
+  get_filename_component(_ts_source_we ${_ts_source} NAME_WE)
+  set(_prefix ${CMAKE_CURRENT_BINARY_DIR}/${_ts_source_we})
+  list(APPEND ${PROJECT_NAME}_JS_PROTOS ${_prefix}.js ${_prefix}.js.map)
+  list(APPEND ${PROJECT_NAME}_TS_PROTOS ${_prefix}.d.ts ${_prefix}.d.ts.map)
+  list(APPEND ${PROJECT_NAME}_FS_PROTOS ${_prefix}.fs)
+  add_custom_command(
+    OUTPUT ${_prefix}.fs
+    WORKING_DIRECTORY ${CMAKE_CURRENT_SOURCE_DIR}
+    COMMAND npx --no-install ts2fable ${_prefix}.d.ts ${_prefix}.fs
+    DEPENDS ${_prefix}.d.ts
+  )
+endforeach()

 add_custom_command(
-  OUTPUT ${CMAKE_CURRENT_BINARY_DIR}/bundle.js
-  COMMAND ${_quiet_exec}
-  ARGS
-  ${_venv_exec}
-  ${_venv_python}
-  ${_build_admin_web_app_py}
-  --protos ${_proto_abs_SOURCES}
-  --tsconfig ${_tsconfig}
-  --mode ${WEBPACK_MODE}
-  --output-dir ${CMAKE_CURRENT_BINARY_DIR}
-  COMMENT "Building Spark.Admin.WebApp"
-  DEPENDS ${_build_admin_web_app_deps})
+  OUTPUT
+    ${${PROJECT_NAME}_JS_PROTOS}
+    ${${PROJECT_NAME}_TS_PROTOS}
+  WORKING_DIRECTORY ${CMAKE_CURRENT_SOURCE_DIR}
+  COMMAND npx
+    --no-install tsc
+    -p ${_tsconfig}
+  DEPENDS
+    ${_tsconfig}
+    ${CMAKE_CURRENT_BINARY_DIR}/proto.js
+)
+add_custom_target(
+  transpile_type_script
+  ALL
+  DEPENDS
+    ${${PROJECT_NAME}_JS_PROTOS}
+    ${${PROJECT_NAME}_TS_PROTOS}
+)
+add_custom_target(generate_fsts_stubs ALL DEPENDS ${${PROJECT_NAME}_FS_PROTOS})

-add_custom_target(admin_web_app_bundle ALL DEPENDS ${CMAKE_CURRENT_BINARY_DIR}/bundle.js)
+sym_link_dir(
+  TARGET_NAME link_webapp_public
+  SOURCE ${CMAKE_CURRENT_SOURCE_DIR}/public
+  DEST ${CMAKE_CURRENT_BINARY_DIR}/public
+)
+sym_link_dir(
+  TARGET_NAME link_webapp_root
+  SOURCE ${CMAKE_CURRENT_BINARY_DIR}
+  DEST ${CMAKE_SOURCE_DIR}/build/Spark.Admin.WebApp
+)
+
+file(
+  GLOB_RECURSE ${PROJECT_NAME}_FS_SOURCES
+  RELATIVE ${CMAKE_CURRENT_SOURCE_DIR}
+  src/*.fs
+)
+add_custom_command(
+  OUTPUT
+    ${CMAKE_CURRENT_BINARY_DIR}/bundle.js
+    $<$<CONFIG:DEBUG>:${CMAKE_CURRENT_BINARY_DIR}/bundle.js.map>
+  WORKING_DIRECTORY ${CMAKE_CURRENT_SOURCE_DIR}/src
+  COMMAND dotnet restore
+  COMMAND dotnet tool restore
+  COMMAND export SPARK_ADMIN_WEB_APP_OUTPUT_DIR=${CMAKE_CURRENT_BINARY_DIR}
+  COMMAND dotnet fable -o ${CMAKE_CURRENT_BINARY_DIR} --verbose
+  COMMAND npm run build -- --mode $<IF:$<CONFIG:Debug>,"development","production">
+  DEPENDS
+    ${${PROJECT_NAME}_FS_SOURCES}
+    ${${PROJECT_NAME}_FS_PROTOS}
+    ${CMAKE_CURRENT_SOURCE_DIR}/src/Spark.Admin.WebApp.fsproj
+)
+add_custom_target(
+  bundle_spark_admin_webapp ALL
+  DEPENDS
+    ${CMAKE_CURRENT_BINARY_DIR}/bundle.js
+    $<$<CONFIG:DEBUG>:${CMAKE_CURRENT_BINARY_DIR}/bundle.js.map>
+  COMMENT "Bundling Spark.Admin.WebApp"
+)
+
+# Exclude 'public' dir because it is a symlink
+install(
+  DIRECTORY "${CMAKE_CURRENT_BINARY_DIR}"
+  DESTINATION "${CMAKE_INSTALL_BINDIR}"
+  PATTERN "public" EXCLUDE
+)
+# Install actual (non-symlinked) 'public' dir
+install(
+  DIRECTORY "${CMAKE_CURRENT_SOURCE_DIR}/public"
+  DESTINATION "${CMAKE_INSTALL_BINDIR}/${PROJECT_NAME}"
+)

-install(DIRECTORY "${CMAKE_CURRENT_BINARY_DIR}" DESTINATION "${SPARK_INSTALL_PREFIX}/bin")
@@ -42,7 +42,7 @@
    "ts-loader": "^9.4.2",
    "tsutils": "^3.21.0",
    "typescript": "^4.9.5",
-    "webpack-cli": "^5.0.1",
+    "webpack-cli": "^5.1.4",
    "webpack-dev-server": "^4.11.1",
    "ws": "^8.12.1"
  }
@@ -17,7 +17,7 @@ type Tab =
  | PositionManagementTab
  | MarginTab

-let venues = [| Venue.BINANCE; Venue.BinanceUs; Venue.BinanceUsdsFutures; Venue.CBOE; Venue.COINBASE; Venue.DERIBIT; Venue.GLBX; Venue.ICE; Venue.TAIFEX; Venue.SIM; |]
+let venues = [| Venue.BINANCE; Venue.BinanceUs; Venue.BinanceUsdsFutures; Venue.CBOE; Venue.COINBASE; Venue.DERIBIT; Venue.OKX; Venue.GLBX; Venue.ICE; Venue.TAIFEX; Venue.SIM; |]

 let securityTypes = [| SecurityType.SPOT; SecurityType.FUTURE; SecurityType.OPTION; SecurityType.SPREAD; |]

@@ -74,6 +74,11 @@ type TaifexExchangeField =
  | Origin of CustomerOrFirm
  | ClearingAccount of string

+type OkxExchangeField =
+  | CTI of CTICode
+  | Origin of CustomerOrFirm
+  | ClearingAccount of string
+
 type SIMExchangeField =
  | CTI of CTICode
  | Origin of CustomerOrFirm
@@ -6,6 +6,7 @@ using mbts.ui.controls;
 using mbts.ui.converters;
 using mbts.ui.extensions;

+using System;
 using System.Collections.Generic;
 using System.Linq;
 using System.Reactive.Linq;
@@ -127,7 +128,11 @@ namespace mbts.ui.spark {
            size,
            Observable.Return(calc_buttons));

-         _show_popup(ticket_vm, new OrderTicketHeader { bbo = ticket_vm.bbo }, resources);
+         _show_popup(
+            ticket_vm,
+            new OrderTicketHeader { bbo = ticket_vm.bbo },
+            resources,
+            view => Behavior.SetAutoFocusTargetProvider(view, AutoFocusTargetProvider.TypeProvider(typeof(TextBox), view)));
      }

      void _show_popup(object content, object header) {
@@ -136,7 +141,7 @@ namespace mbts.ui.spark {
         _show_popup(content, header, resources);
      }

-      void _show_popup(object content, object header, ResourceDictionary resources) {
+      void _show_popup(object content, object header, ResourceDictionary resources, Action<HeaderedContentControl> init_view = null) {
         var view = new HeaderedContentControl {
            DataContext = content,
            Content = content,
@@ -146,6 +151,7 @@ namespace mbts.ui.spark {
            Resources = resources ?? new AuxiliaryDataResources()
         };
         view.Style = view.Resources[AuxiliaryDataResources.PopupChildStyleKey] as Style;
+         init_view?.Invoke(view);
         _popup_man.show(view);
      }

@@ -10,6 +10,8 @@ using bts.view;

 using mbts.persist;
 using mbts.ui;
+using mbts.ui.controls;
+using mbts.ui.extensions;
 using mbts.ui.settings;
 using mbts.ui.spark;
 using mbts.ui.utility;
@@ -27,12 +29,18 @@ using System.Collections.Immutable;
 using System.Collections.Specialized;
 using System.Linq;
 using System.Reactive.Linq;
+using System.Threading;
 using System.Windows;
+using System.Windows.Input;
+using System.Windows.Threading;

 namespace mbts.spark {
   class MarketViewWindow : SparkWindow<MarketViewSettings>, ISharedWindow, ISlowMeasureContentSize {
      const string _window_type = "MarketView", _window_type_display_name = "Market View";

+      static int _window_id_counter = 0;
+
+      readonly string _window_id = $"{_window_type}_{Interlocked.Increment(ref _window_id_counter)}";
      readonly OrdersModel _orders_model;
      readonly ISecurityMetadataStore _sec_meta_store;
      readonly IPositionModel _position_model;
@@ -122,7 +130,7 @@ namespace mbts.spark {
         Log.notnull(_state);
         _menu_display_vm = new SparkMenuDisplayVM();
         _menu_display_vm.load_from(_state.menu_display);
-         var update_securities_cmd = SparkCommands.change_securities_callback(_window_type_display_name);
+         var update_securities_cmd = SparkCommands.change_securities_callback(_window_id);
         var security_md_map_src = Observable.Create<IReadOnlyDictionary<uint, SecurityMD>>(o => {
            IDisposable sub = null;
            _security_criteria_changed += on_security_criteria_changed;
@@ -286,7 +294,32 @@ namespace mbts.spark {
         }
      }

-      protected override void OnParentViewCreated(SparkWindowView parent_view) =>
+      protected override void OnParentViewCreated(SparkWindowView parent_view) {
         _menu_display_vm.safe_bind_to_view(parent_view);
+         ui.WindowStyle.AddWinFormsHostChangedHandler(parent_view, OnWinFormsHostChanged);
+
+         void OnWinFormsHostChanged(object sender, RoutedPropertyChangedEventArgs<WPFPanel> e) {
+            // If this window is "activated" by the user clicking on a popup window (such as an
+            // order ticket), it never receives keyboard focus (seems like a bug in Windows). As a
+            // workaround we explicitly focus the view when the parent WPFPanel gets focus. If the
+            // user clicks on the popup a second time, it should get keyboard focus.
+            if (e.OldValue is not null) {
+               e.OldValue.GotFocus -= OnGotFocus;
+            }
+            if (e.NewValue is not null) {
+               e.NewValue.GotFocus += OnGotFocus;
+            }
+
+            void OnGotFocus(object sender, EventArgs e) =>
+               _view.Dispatcher.InvokeAsync(StealFocus, DispatcherPriority.Send);
+
+            void StealFocus() {
+               if (_view.IsVisible) {
+                  Log.Ui.info($"MarketViewWindow. Stealing focus from WPFPanel");
+                  _view.Focus();
+               }
+            }
+         }
+      }
   }
 }
@@ -24,6 +24,7 @@
   using System.Diagnostics.CodeAnalysis;
   using System.Linq;
   using System.Reactive.Linq;
+   using System.Threading;
   using System.Windows;

   class TimeAndSalesWindow : SparkWindow<TimeAndSalesSettings>, ISharedWindow {
@@ -35,7 +36,9 @@
      const double _VERTICAL_SCROLLBAR_OVERLAP = 9;

      static readonly TimeSpan _trade_poll_interval = TimeSpan.FromSeconds(0.4);
+      static int _window_id_counter = 0;

+      readonly string _window_id = $"{WINDOW_TYPE_NAME}_{Interlocked.Increment(ref _window_id_counter)}";
      [DisallowNull]
      readonly MDConsumer _consumer;
      [DisallowNull]
@@ -133,7 +136,7 @@
            }
            return result;
         });
-         var update_securities_cmd = SparkCommands.change_securities_callback(WINDOW_DISPLAY_NAME);
+         var update_securities_cmd = SparkCommands.change_securities_callback(_window_id);
         var security_criteria_changed = new Action<SecurityCriteria>(sc => { });

         SparkCommand toggle_is_paused_cmd = null;
@@ -314,6 +314,12 @@
            <TextBlock Text="{Binding param.display_value, Converter={StaticResource DoubleTextConverter}}"/>
          </DataTemplate>
        </usp:AlgoParamTemplateSelector.DoubleTemplate>
+        <!--Nullable Double-->
+        <usp:AlgoParamTemplateSelector.NullableDoubleTemplate>
+          <DataTemplate>
+            <TextBlock Text="{Binding param.display_value, Converter={StaticResource DoubleTextConverter}}"/>
+          </DataTemplate>
+        </usp:AlgoParamTemplateSelector.NullableDoubleTemplate>
        <!--Enum-->
        <usp:AlgoParamTemplateSelector.EnumTemplate>
          <DataTemplate>
@@ -435,6 +441,20 @@
            </c:TickUpDownDouble>
          </DataTemplate>
        </usp:AlgoParamTemplateSelector.DoubleTemplate>
+        <!--Nullable Double-->
+        <usp:AlgoParamTemplateSelector.NullableDoubleTemplate>
+          <DataTemplate>
+            <c:TickUpDownNullableDouble x:Name="tick_up_down" Focusable="False" HorizontalAlignment="Stretch" Value="{Binding param.display_value, UpdateSourceTrigger=PropertyChanged}" ArrowKeyTickModifiers="Ctrl">
+              <c:TickUpDownNullableDouble.CommandBindings>
+                <CommandBinding Command="{x:Static u:NavigationHelper.MoveFocusCommand}" Executed="{x:Static u:NavigationHelper.KeyboardMoveFocusExecuted}"/>
+              </c:TickUpDownNullableDouble.CommandBindings>
+              <c:TickUpDownNullableDouble.InputBindings>
+                <KeyBinding Key="Up" Command="{x:Static u:NavigationHelper.MoveFocusCommand}" CommandParameter="{x:Static FocusNavigationDirection.Up}"/>
+                <KeyBinding Key="Down" Command="{x:Static u:NavigationHelper.MoveFocusCommand}" CommandParameter="{x:Static FocusNavigationDirection.Down}"/>
+              </c:TickUpDownNullableDouble.InputBindings>
+            </c:TickUpDownNullableDouble>
+          </DataTemplate>
+        </usp:AlgoParamTemplateSelector.NullableDoubleTemplate>
        <!--Enum-->
        <usp:AlgoParamTemplateSelector.EnumTemplate>
          <DataTemplate>
@@ -33,6 +33,7 @@ namespace mbts.ui.spark {
      public DataTemplate UIntTemplate { get; set; }
      public DataTemplate NullableUIntTemplate { get; set; }
      public DataTemplate DoubleTemplate { get; set; }
+      public DataTemplate NullableDoubleTemplate { get; set; }
      public DataTemplate EnumTemplate { get; set; }
      public DataTemplate NullableEnumTemplate { get; set; }
      public DataTemplate StringTemplate { get; set; }
@@ -52,6 +53,7 @@ namespace mbts.ui.spark {
         IAlgoParamVM pvm = item as IAlgoParamVM ?? (item as AlgoParamRowVM)?.param;
         if (pvm != null) {
            if (pvm is IAlgoParamVM<double>) return DoubleTemplate;
+            if (pvm is IAlgoParamVM<double?>) return NullableDoubleTemplate;
            if (pvm is IAlgoParamVM<int>) return IntTemplate;
            if (pvm is IAlgoParamVM<int?>) return NullableIntTemplate;
            if (pvm is IAlgoParamVM<uint>) return UIntTemplate;
@@ -62,6 +62,7 @@ namespace mbts.spark {
      readonly RefValueContainer<IEnumerable<VolumeAtPriceLevelImmutable>> _volume_container = new();

      IDisposable _settings_sub, _volume_sub, _tick_sub, _fair_value_lines_sub, _vm_sub;
+      readonly ThrottledAction _scroll_to_tick_log_throttle = new(TimeSpan.FromSeconds(5));
      readonly SingleValuePump<ScrollViewer> _scroll_viewer_source;

      ScrollViewer _scroll_viewer {
@@ -452,8 +453,8 @@ namespace mbts.spark {

      public bool scroll_to_tick(long tick) {
         if (tick > int.MaxValue) {
-            Log.Ui.error(
-               $"Unable to scroll to tick {tick} because it exceeds int.MaxValue");
+            _scroll_to_tick_log_throttle.Execute(() =>
+               Log.Ui.error($"Unable to scroll to tick {tick} because it exceeds int.MaxValue"));
            return false;
         }
         return scroll_to_tick((int)tick);
@@ -79,13 +79,6 @@ namespace mbts.spark {
      public static int index_of(IList<double> prices, double price, IComparer<double> comparer, bool relative_idx) {
         for (int i = 1; i < prices.Count; i++) {
            Log.assert(prices[i - 1] > prices[i]);
-            var c = comparer.Compare(prices[i - 1], prices[i]);
-            if (c >= 0) {
-               c = comparer.Compare(prices[i - 1], prices[i]);
-            } else {
-               c = comparer.Compare(prices[i - 1], prices[i]);
-            }
-            Log.assert(c < 0);
         }
         var idx = CollectionHelper.binary_search(prices, price, comparer);
         if (relative_idx && idx < 0) {
@@ -254,8 +247,8 @@ namespace mbts.spark {
               var price = ticks.HasValue ? spec.tickup(0, ticks.Value) : rung.price;
               // prices are in descending order, so get diff from curr_price - price
               var tick_diff = spec.tickdiff(curr_price, price).assert_value();
-               checked {
-                  curr_index += (int)tick_diff;
+               if (!MathUtils.try_increment(ref curr_index, tick_diff)) {
+                  continue;
               }
               curr_price = price;
               if (curr_index >= 0 && curr_index < prices.Count) {
@@ -289,7 +289,8 @@ namespace bts.shell.pnl {

      public AccountTuple account => new(account_id, account_name);
      public string symbol => security?.pnl_symbol() ?? string.Empty;
-      public DateTime expiration => security?.earliest_expiration() ?? DateTime.MinValue;
+      public DateTime expiration =>
+         security?.normal_expirations()?.MinOrDefault(DateTime.MinValue) ?? DateTime.MinValue;
      public double effective_cpp => override_cpp ?? default_cpp;

      /// Market/Calc
@@ -398,7 +398,7 @@
            $"Expected rounded qty to be real, non-negative.  " +
            $"qty: {qty}; rounded_qty: {rounded_qty}; display: {display})");
         var iqty = Math.Floor(rounded_qty);
-         return ((uint)iqty, new() { Fraction = Math.Round(rounded_qty - iqty, 2) });
+         return ((uint)iqty, new() { Fraction = Math.Round(rounded_qty - iqty, 6) });
      }

      private CustomOrderDefinitionImmutable? _create_custom_def_for_stop_and_stop_limit(OrderRecordImmutable order, ref double stop_price) {
@@ -1,41 +0,0 @@
-// using System;
-// using System.Collections.Generic;
-// using System.Linq;
-// using System.Text;
-
-// namespace BCG
-// {
-//    public class DateTimeFormatter
-//    {
-//       private delegate string MyFormatDateTime (DateTime v);
-//       public enum ExpirationShortForms
-//       {
-//          DayMonthYearDiff2000,
-//          MonthYearDiff2000
-//       }
-//       public enum DateTimeFormats { Regular, ShortDate, DayMonthYearDiff2000, MonthYearDiff2000 }
-//       private static Dictionary<string, MyFormatDateTime> _local_formats = null;
-//       private static void _init()
-//       {
-//          if (_local_formats == null)
-//          {
-//             _local_formats = new Dictionary<string, MyFormatDateTime>();
-//             _local_formats.Add(DateTimeFormats.Regular.ToString(), v => v.ToString());
-//             _local_formats.Add(DateTimeFormats.ShortDate.ToString(), v => v.ToShortDateString());
-//             _local_formats.Add(DateTimeFormats.DayMonthYearDiff2000.ToString(),
-//                (DateTime v) => { return String.Format("{0} {1}{2}", v.Day, BCG.Utilities.Dates.DateCode.getMonthSymbol(v.Month), v.Year - 2000); });
-//             _local_formats.Add(DateTimeFormats.MonthYearDiff2000.ToString(),
-//                (DateTime v) => { return String.Format("{0}{1}", BCG.Utilities.Dates.DateCode.getMonthSymbol(v.Month), v.Year - 2000); });            
-//          }
-//       }
-//       public static string format(DateTime val, string format)
-//       {
-//          _init ();
-//          MyFormatDateTime f = null;
-//          if (_local_formats.TryGetValue(format, out f))
-//             return f(val);
-//          else
-//             return val.ToString(format);
-//       }
-//    }
-// }
@@ -20,13 +20,23 @@ namespace mbts.ui {
   }

   public static class AutoFocusTargetProvider {
-      public static IAutoFocusTargetProvider<T> TypeProvider<T>(FrameworkElement root, bool focus, bool focus_keyboard, bool release_keyboard, bool capture_mouse, bool release_mouse) where T : FrameworkElement =>
-         new AnonymousProvider<T>(() => root.descendants_of_type<T>().FirstOrDefault(), focus, focus_keyboard, release_keyboard, capture_mouse, release_mouse);
-
-      public static IAutoFocusTargetProvider TypeProvider(Type type, FrameworkElement root, bool focus, bool focus_keyboard, bool release_keybobard, bool capture_mouse, bool release_mouse) =>
+      public static IAutoFocusTargetProvider TypeProvider(
+         Type type,
+         FrameworkElement root,
+         bool focus = true,
+         bool focus_keyboard = false,
+         bool release_keyboard = true,
+         bool capture_mouse = false,
+         bool release_mouse = true) =>
         new AnonymousProvider<FrameworkElement>(
-            () => root.visual_children_where(el => type.IsAssignableFrom(el.GetType())).Cast<FrameworkElement>().FirstOrDefault(),
-            focus, focus_keyboard, release_keybobard, capture_mouse, release_mouse);
+            () => root.visual_children_where(el => type.IsAssignableFrom(el.GetType()))
+               .Cast<FrameworkElement>()
+               .FirstOrDefault(),
+            focus,
+            focus_keyboard,
+            release_keyboard,
+            capture_mouse,
+            release_mouse);

      private class AnonymousProvider<T> : IAutoFocusTargetProvider<T> where T : FrameworkElement {
         readonly Func<T> _get_target;
@@ -1,248 +1,264 @@
-using bts.utils;
-
-using mbts.utils;
-
-using Spark.Core.Disposables;
-using Spark.ViewModel.Commands;
-
-using System;
-using System.Windows;
-using System.Windows.Controls;
-using System.Windows.Controls.Primitives;
-using System.Windows.Data;
-using System.Windows.Input;
-
-namespace mbts.ui {
-   static class Behavior {
-      #region CloseWindow
-      public static readonly DependencyProperty CloseWindowProperty = DependencyProperty.RegisterAttached("CloseWindow", typeof(bool), typeof(Behavior),
-         new PropertyMetadata(false, (d, dpc_args) => {
-            var button = d as Button;
-            if (button != null && (bool)dpc_args.NewValue) {
-               RoutedEventHandler close_window = (sender, e) => {
-                  var window = button.find_ancestor_of_type<Window>();
-                  if (window != null)
-                     window.Close();
-               };
-               button.Click += close_window;
-               IDisposable dispose = Disposable.Create(() => {
-                  button.Click -= close_window;
-                  close_window = null;
-               });
-               IDisposable early_dispose = button.ObserveProperty(CloseWindowProperty, (sender, e) => {
-                  if (!GetCloseWindow(button))
-                     Disposable.dispose(ref dispose);
-               });
-               DisposeExtensions.attach_to_disposer(button, dispose);
-               DisposeExtensions.attach_to_disposer(button, early_dispose);
-            }
-         }));
-      public static bool GetCloseWindow(Button obj) { return (bool)obj.GetValue(CloseWindowProperty); }
-      public static void SetCloseWindow(Button obj, bool value) { obj.SetValue(CloseWindowProperty, value); }
-      #endregion
-
-      #region AnimatedScrollViewerVerticalOffset
-      static double GetAnimatedScrollViewerVerticalOffset(DependencyObject obj) =>
-         (double)obj.GetValue(AnimatedScrollViewerVerticalOffsetProperty);
-
-      public static void SetAnimatedScrollViewerVerticalOffset(DependencyObject obj, double value) =>
-         obj.SetValue(AnimatedScrollViewerVerticalOffsetProperty, value);
-
-      public static readonly DependencyProperty AnimatedScrollViewerVerticalOffsetProperty =
-          DependencyProperty.RegisterAttached("AnimatedScrollViewerVerticalOffset", typeof(double), typeof(Behavior), new PropertyMetadata(0.0, scroll_changed));
-
-      static void scroll_changed(DependencyObject d, DependencyPropertyChangedEventArgs e) {
-         ScrollViewer target = d as ScrollViewer;
-         if (target != null) {
-            target.ScrollToVerticalOffset((double)e.NewValue);
-         }
-      }
-      #endregion
-
-      #region AutoFocusTargetProvider
-      public static readonly DependencyProperty AutoFocusTargetProviderProperty = DependencyProperty.RegisterAttached("AutoFocusTargetProvider", typeof(bool),
-         typeof(Behavior), new PropertyMetadata(OnAutoFocusTargetProviderChanged));
-
-      static void OnAutoFocusTargetProviderChanged(DependencyObject d, DependencyPropertyChangedEventArgs e) {
-         if (e.NewValue is IAutoFocusTargetProvider aftp) {
-            var el = ExceptionAssert.Argument.Is<FrameworkElement>(d, nameof(d));
-            d.SetValue(AutoFocusTargetProviderLifeDependencyPropertyKey, new AutoFocusTargetProviderLifeDependency(el, aftp));
-         } else
-            d.SetValue(AutoFocusTargetProviderLifeDependencyPropertyKey, null);
-      }
-
-      public static bool GetAutoFocusTargetProvider(DependencyObject d) => (bool)d.GetValue(AutoFocusTargetProviderProperty);
-      public static void SetAutoFocusTargetProvider(DependencyObject d, bool value) => d.SetValue(AutoFocusTargetProviderProperty, value);
-      #endregion
-
-      #region AutoFocusTargetProviderLifeDependency (ReadOnly)
-      static readonly DependencyPropertyKey AutoFocusTargetProviderLifeDependencyPropertyKey = DependencyProperty.RegisterAttachedReadOnly("AutoFocusTargetProviderLifeDependency",
-         typeof(FrameworkElementLifeDependency), typeof(Behavior), new PropertyMetadata(FrameworkElementLifeDependencyHelper.DisposeOldInitializeNew));
-
-      public static readonly DependencyProperty AutoFocusTargetProviderLifeDependencyProperty = AutoFocusTargetProviderLifeDependencyPropertyKey.DependencyProperty;
-
-      public static FrameworkElementLifeDependency GetAutoFocusTargetProviderLifeDependency(DependencyObject d) => d.GetValue(AutoFocusTargetProviderLifeDependencyProperty) as FrameworkElementLifeDependency;
-      public static void SetAutoFocusTargetProviderLifeDependency(DependencyObject d, FrameworkElementLifeDependency value) => d.SetValue(AutoFocusTargetProviderLifeDependencyPropertyKey, value);
-
-      class AutoFocusTargetProviderLifeDependency : FrameworkElementLifeDependency {
-         readonly IAutoFocusTargetProvider _focus_target_provider;
-
-         public AutoFocusTargetProviderLifeDependency(FrameworkElement root, IAutoFocusTargetProvider focus_target_provider) : base(root) =>
-            _focus_target_provider = focus_target_provider;
-
-         protected override void OnLoaded() {
-            var el = _focus_target_provider.Target;
-            if (el != null) {
-               if (_focus_target_provider.Focus)
-                  el.Focus();
-               if (_focus_target_provider.FocusKeyboard)
-                  Keyboard.Focus(el);
-               if (_focus_target_provider.CaptureMouse)
-                  el.CaptureMouse();
-            }
-         }
-
-         protected override void OnUnloaded() {
-            var el = _focus_target_provider.Target;
-            if (el != null) {
-               if (_focus_target_provider.ReleaseKeyboard) {
-                  ReferenceEquals(Keyboard.FocusedElement, el);
-                  Keyboard.ClearFocus();
-               }
-               if (_focus_target_provider.ReleaseMouse)
-                  el.ReleaseMouseCapture();
-            }
-         }
-      }
-      #endregion
-
-      #region EatMouseDown
-      public static readonly DependencyProperty EatMouseDownProperty =
-         DependencyProperty.RegisterAttached("EatMouseDown", typeof(bool), typeof(Behavior), new PropertyMetadata(false, OnEatMouseDownChanged));
-
-      static void OnEatMouseDownChanged(DependencyObject d, DependencyPropertyChangedEventArgs e) {
-         var el = (FrameworkElement)d;
-         if ((bool)e.NewValue) el.MouseDown += _eat_mouse_down;
-         else el.MouseDown -= _eat_mouse_down;
-      }
-
-      static void _eat_mouse_down(object sender, MouseButtonEventArgs e) =>
-         e.Handled = true;
-
-      public static bool GetEatMouseDown(DependencyObject d) => (bool)d.GetValue(EatMouseDownProperty);
-      public static void SetEatMouseDown(DependencyObject d, bool value) => d.SetValue(EatMouseDownProperty, value);
-      #endregion
-
-      #region DisablePaste
-      public static readonly DependencyProperty DisablePasteProperty = DependencyProperty.RegisterAttached(
-         "DisablePaste", typeof(bool), typeof(Behavior), new PropertyMetadata(false, OnDisablePasteChanged));
-
-      static void OnDisablePasteChanged(DependencyObject d, DependencyPropertyChangedEventArgs e) {
-         if ((bool)e.NewValue)
-            DataObject.AddPastingHandler(d, _disable_paste_pasting_handler);
-         else DataObject.RemovePastingHandler(d, _disable_paste_pasting_handler);
-      }
-      static void _disable_paste_pasting_handler(object sender, DataObjectPastingEventArgs e) => e.CancelCommand();
-
-      public static bool GetDisablePaste(DependencyObject d) => (bool)d.GetValue(DisablePasteProperty);
-      public static void SetDisablePaste(DependencyObject d, bool value) => d.SetValue(DisablePasteProperty, value);
-      #endregion
-
-      public static readonly DependencyProperty BindingUpdateGroupsProperty =
-         DependencyProperty.RegisterAttached("BindingUpdateGroups", typeof(BindingUpdateGroupCollection), typeof(Behavior), new PropertyMetadata(null));
-
-      public static BindingUpdateGroupCollection GetBindingUpdateGroups(DependencyObject d) => d.GetValue(BindingUpdateGroupsProperty) as BindingUpdateGroupCollection;
-      public static void SetBindingUpdateGroups(DependencyObject d, BindingUpdateGroupCollection value) => d.SetValue(BindingUpdateGroupsProperty, value);
-
-      #region AutoRaiseCanExecuteChanged
-      public static readonly DependencyProperty AutoRaiseCanExecuteChangedProperty =
-         DependencyProperty.RegisterAttached("AutoRaiseCanExecuteChanged", typeof(bool), typeof(Behavior),
-            new PropertyMetadata(false, (d, e) => {
-               if ((bool)e.NewValue) {
-                  BindingOperations.SetBinding(d, AutoCommandProperty, new Binding(nameof(ButtonBase.Command)) { Source = d, Mode = BindingMode.OneWay });
-                  BindingOperations.SetBinding(d, AutoCommandParameterProperty, new Binding(nameof(ButtonBase.CommandParameter)) { Source = d, Mode = BindingMode.OneWay });
-               } else {
-                  BindingOperations.ClearBinding(d, AutoCommandProperty);
-                  BindingOperations.ClearBinding(d, AutoCommandParameterProperty);
-               }
-            }));
-
-      public static bool GetAutoRaiseCanExecuteChanged(DependencyObject d) => (bool)d.GetValue(AutoRaiseCanExecuteChangedProperty);
-      public static void SetAutoRaiseCanExecuteChanged(DependencyObject d, bool value) => d.SetValue(AutoRaiseCanExecuteChangedProperty, value);
-
-      /// Do not bind directly to <see cref="AutoCommandProperty"/> (used internally by <see cref="AutoRaiseCanExecuteChangedProperty"/>
-      public static readonly DependencyProperty AutoCommandProperty =
-         DependencyProperty.RegisterAttached("AutoCommand", typeof(ICommand), typeof(Behavior), new PropertyMetadata(null, _try_raise_can_execute));
-
-      public static ICommand GetAutoCommand(DependencyObject d) => d.GetValue(AutoCommandProperty) as ICommand;
-      public static void SetAutoCommand(DependencyObject d, ICommand value) => d.SetValue(AutoCommandProperty, value);
-
-      /// Do not bind directly to <see cref="AutoCommandParameterProperty"/> (used internally by <see cref="AutoRaiseCanExecuteChangedProperty"/>
-      public static readonly DependencyProperty AutoCommandParameterProperty =
-         DependencyProperty.RegisterAttached("AutoCommandParameter", typeof(object), typeof(Behavior), new PropertyMetadata(null, _try_raise_can_execute));
-
-      public static object GetAutoCommandParameter(DependencyObject d) => d.GetValue(AutoCommandParameterProperty);
-      public static void SetAutoCommandParameter(DependencyObject d, object value) => d.SetValue(AutoCommandParameterProperty, value);
-
-      static void _try_raise_can_execute(DependencyObject d, DependencyPropertyChangedEventArgs e) =>
-            (d.GetValue(ButtonBase.CommandProperty) as IRaiseCanExecuteChangedCommand)?.RaiseCanExecuteChanged();
-      #endregion
-
-      #region FocusOnLoaded
-      public static readonly DependencyProperty FocusOnLoadedProperty =
-         DependencyProperty.RegisterAttached("FocusOnLoaded", typeof(bool), typeof(Behavior), new PropertyMetadata(false, OnFocusOnLoadedChanged));
-
-      static void OnFocusOnLoadedChanged(DependencyObject d, DependencyPropertyChangedEventArgs e) {
-         var target = (FrameworkElement)d;
-         if ((bool)e.NewValue) target.Loaded += _target_Loaded;
-         else target.Loaded -= _target_Loaded;
-      }
-
-      static void _target_Loaded(object sender, RoutedEventArgs e) {
-         var target = (FrameworkElement)sender;
-         Keyboard.Focus(target);
-      }
-      public static bool GetFocusOnLoaded(DependencyObject d) => (bool)d.GetValue(FocusOnLoadedProperty);
-      public static void SetFocusOnLoaded(DependencyObject d, bool value) => d.SetValue(FocusOnLoadedProperty, value);
-      #endregion
-
-      #region MirrorLayout
-      static readonly DependencyProperty[] _layout_props = new[] {
-         FrameworkElement.HeightProperty,
-         FrameworkElement.WidthProperty,
-         FrameworkElement.MarginProperty,
-         Control.PaddingProperty,
-         FrameworkElement.HorizontalAlignmentProperty,
-         Control.HorizontalContentAlignmentProperty,
-         FrameworkElement.VerticalAlignmentProperty,
-         Control.VerticalContentAlignmentProperty
-      };
-
-      public static readonly DependencyProperty MirrorLayoutForProperty =
-         DependencyProperty.RegisterAttached("MirrorLayoutFor", typeof(FrameworkElement), typeof(Behavior),
-            new PropertyMetadata(OnMirrorLayoutForChanged));
-
-      static void OnMirrorLayoutForChanged(DependencyObject d, DependencyPropertyChangedEventArgs e) {
-         var target = new LazyCast<FrameworkElement, DependencyObject>(d);
-         if (e.OldValue is FrameworkElement old_src) {
-            target.value.clear_bindings(_layout_props);
-         }
-         if (e.NewValue is FrameworkElement new_src) {
-            target.value.bind(FrameworkElement.WidthProperty, new_src, FrameworkElement.ActualWidthProperty);
-            target.value.bind(FrameworkElement.HeightProperty, new_src, FrameworkElement.ActualHeightProperty);
-            target.value.bind(FrameworkElement.MarginProperty, new_src, FrameworkElement.MarginProperty);
-            target.value.bind(FrameworkElement.HorizontalAlignmentProperty, new_src, FrameworkElement.HorizontalAlignmentProperty);
-            target.value.bind(FrameworkElement.VerticalAlignmentProperty, new_src, FrameworkElement.VerticalAlignmentProperty);
-            if (target.value is Control && new_src is Control) {
-               target.value.bind(Control.PaddingProperty, new_src, Control.PaddingProperty);
-               target.value.bind(Control.HorizontalContentAlignmentProperty, new_src, Control.HorizontalContentAlignmentProperty);
-               target.value.bind(Control.VerticalContentAlignmentProperty, new_src, Control.VerticalContentAlignmentProperty);
-            }
-         }
-      }
-
-      public static FrameworkElement GetMirrorLayoutFor(DependencyObject d) => d.GetValue(MirrorLayoutForProperty) as FrameworkElement;
-      public static void SetMirrorLayoutFor(DependencyObject d, FrameworkElement value) => d.SetValue(MirrorLayoutForProperty, value);
-      #endregion
-   }
-}
+using bts.utils;
+
+using mbts.utils;
+
+using Spark.Core.Disposables;
+using Spark.ViewModel.Commands;
+
+using System;
+using System.Windows;
+using System.Windows.Controls;
+using System.Windows.Controls.Primitives;
+using System.Windows.Data;
+using System.Windows.Input;
+
+namespace mbts.ui {
+   static class Behavior {
+      #region CloseWindow
+      public static readonly DependencyProperty CloseWindowProperty = DependencyProperty.RegisterAttached("CloseWindow", typeof(bool), typeof(Behavior),
+         new PropertyMetadata(false, (d, dpc_args) => {
+            var button = d as Button;
+            if (button != null && (bool)dpc_args.NewValue) {
+               RoutedEventHandler close_window = (sender, e) => {
+                  var window = button.find_ancestor_of_type<Window>();
+                  if (window != null)
+                     window.Close();
+               };
+               button.Click += close_window;
+               IDisposable dispose = Disposable.Create(() => {
+                  button.Click -= close_window;
+                  close_window = null;
+               });
+               IDisposable early_dispose = button.ObserveProperty(CloseWindowProperty, (sender, e) => {
+                  if (!GetCloseWindow(button))
+                     Disposable.dispose(ref dispose);
+               });
+               DisposeExtensions.attach_to_disposer(button, dispose);
+               DisposeExtensions.attach_to_disposer(button, early_dispose);
+            }
+         }));
+      public static bool GetCloseWindow(Button obj) { return (bool)obj.GetValue(CloseWindowProperty); }
+      public static void SetCloseWindow(Button obj, bool value) { obj.SetValue(CloseWindowProperty, value); }
+      #endregion
+
+      #region AnimatedScrollViewerVerticalOffset
+      static double GetAnimatedScrollViewerVerticalOffset(DependencyObject obj) =>
+         (double)obj.GetValue(AnimatedScrollViewerVerticalOffsetProperty);
+
+      public static void SetAnimatedScrollViewerVerticalOffset(DependencyObject obj, double value) =>
+         obj.SetValue(AnimatedScrollViewerVerticalOffsetProperty, value);
+
+      public static readonly DependencyProperty AnimatedScrollViewerVerticalOffsetProperty =
+          DependencyProperty.RegisterAttached("AnimatedScrollViewerVerticalOffset", typeof(double), typeof(Behavior), new PropertyMetadata(0.0, scroll_changed));
+
+      static void scroll_changed(DependencyObject d, DependencyPropertyChangedEventArgs e) {
+         ScrollViewer target = d as ScrollViewer;
+         if (target != null) {
+            target.ScrollToVerticalOffset((double)e.NewValue);
+         }
+      }
+      #endregion
+
+      #region AutoFocusTargetProvider
+      public static readonly DependencyProperty AutoFocusTargetProviderProperty = DependencyProperty.RegisterAttached(
+         "AutoFocusTargetProvider", typeof(IAutoFocusTargetProvider), typeof(Behavior), new PropertyMetadata(OnAutoFocusTargetProviderChanged));
+
+      static void OnAutoFocusTargetProviderChanged(DependencyObject d, DependencyPropertyChangedEventArgs e) {
+         if (e.NewValue is IAutoFocusTargetProvider aftp) {
+            var el = ExceptionAssert.Argument.Is<FrameworkElement>(d, nameof(d));
+            d.SetValue(AutoFocusTargetProviderLifeDependencyPropertyKey, new AutoFocusTargetProviderLifeDependency(el, aftp));
+         } else
+            d.SetValue(AutoFocusTargetProviderLifeDependencyPropertyKey, null);
+      }
+
+      public static IAutoFocusTargetProvider GetAutoFocusTargetProvider(DependencyObject d) =>
+         d.GetValue(AutoFocusTargetProviderProperty) as IAutoFocusTargetProvider;
+
+      public static void SetAutoFocusTargetProvider(DependencyObject d, IAutoFocusTargetProvider value) =>
+         d.SetValue(AutoFocusTargetProviderProperty, value);
+      #endregion
+
+      #region AutoFocusTargetProviderLifeDependency (ReadOnly)
+      static readonly DependencyPropertyKey AutoFocusTargetProviderLifeDependencyPropertyKey = DependencyProperty.RegisterAttachedReadOnly("AutoFocusTargetProviderLifeDependency",
+         typeof(FrameworkElementLifeDependency), typeof(Behavior), new PropertyMetadata(FrameworkElementLifeDependencyHelper.DisposeOldInitializeNew));
+
+      public static readonly DependencyProperty AutoFocusTargetProviderLifeDependencyProperty = AutoFocusTargetProviderLifeDependencyPropertyKey.DependencyProperty;
+
+      public static FrameworkElementLifeDependency GetAutoFocusTargetProviderLifeDependency(DependencyObject d) => d.GetValue(AutoFocusTargetProviderLifeDependencyProperty) as FrameworkElementLifeDependency;
+      public static void SetAutoFocusTargetProviderLifeDependency(DependencyObject d, FrameworkElementLifeDependency value) => d.SetValue(AutoFocusTargetProviderLifeDependencyPropertyKey, value);
+
+      class AutoFocusTargetProviderLifeDependency : FrameworkElementLifeDependency {
+         readonly IAutoFocusTargetProvider _focus_target_provider;
+
+         public AutoFocusTargetProviderLifeDependency(FrameworkElement root, IAutoFocusTargetProvider focus_target_provider) : base(root) =>
+            _focus_target_provider = focus_target_provider;
+
+         protected override void OnLoaded() {
+            var el = _focus_target_provider.Target;
+            if (el != null) {
+               if (_focus_target_provider.Focus) {
+                  var fs = FocusManager.GetFocusScope(el);
+                  if (fs is not null) {
+                     FocusManager.SetFocusedElement(fs, el);
+                     if (fs is UIElement fsUiEl) {
+                        fsUiEl.Focus();
+                     } else if (fs is IInputElement fsInputEl) {
+                        Keyboard.Focus(fsInputEl);
+                     } else {
+                        el.Focus();
+                     }
+                  } else {
+                     el.Focus();
+                  }
+               }
+               if (_focus_target_provider.FocusKeyboard)
+                  Keyboard.Focus(el);
+               if (_focus_target_provider.CaptureMouse)
+                  el.CaptureMouse();
+            }
+         }
+
+         protected override void OnUnloaded() {
+            var el = _focus_target_provider.Target;
+            if (el != null) {
+               if (_focus_target_provider.ReleaseKeyboard) {
+                  ReferenceEquals(Keyboard.FocusedElement, el);
+                  Keyboard.ClearFocus();
+               }
+               if (_focus_target_provider.ReleaseMouse)
+                  el.ReleaseMouseCapture();
+            }
+         }
+      }
+      #endregion
+
+      #region EatMouseDown
+      public static readonly DependencyProperty EatMouseDownProperty =
+         DependencyProperty.RegisterAttached("EatMouseDown", typeof(bool), typeof(Behavior), new PropertyMetadata(false, OnEatMouseDownChanged));
+
+      static void OnEatMouseDownChanged(DependencyObject d, DependencyPropertyChangedEventArgs e) {
+         var el = (FrameworkElement)d;
+         if ((bool)e.NewValue) el.MouseDown += _eat_mouse_down;
+         else el.MouseDown -= _eat_mouse_down;
+      }
+
+      static void _eat_mouse_down(object sender, MouseButtonEventArgs e) =>
+         e.Handled = true;
+
+      public static bool GetEatMouseDown(DependencyObject d) => (bool)d.GetValue(EatMouseDownProperty);
+      public static void SetEatMouseDown(DependencyObject d, bool value) => d.SetValue(EatMouseDownProperty, value);
+      #endregion
+
+      #region DisablePaste
+      public static readonly DependencyProperty DisablePasteProperty = DependencyProperty.RegisterAttached(
+         "DisablePaste", typeof(bool), typeof(Behavior), new PropertyMetadata(false, OnDisablePasteChanged));
+
+      static void OnDisablePasteChanged(DependencyObject d, DependencyPropertyChangedEventArgs e) {
+         if ((bool)e.NewValue)
+            DataObject.AddPastingHandler(d, _disable_paste_pasting_handler);
+         else DataObject.RemovePastingHandler(d, _disable_paste_pasting_handler);
+      }
+      static void _disable_paste_pasting_handler(object sender, DataObjectPastingEventArgs e) => e.CancelCommand();
+
+      public static bool GetDisablePaste(DependencyObject d) => (bool)d.GetValue(DisablePasteProperty);
+      public static void SetDisablePaste(DependencyObject d, bool value) => d.SetValue(DisablePasteProperty, value);
+      #endregion
+
+      public static readonly DependencyProperty BindingUpdateGroupsProperty =
+         DependencyProperty.RegisterAttached("BindingUpdateGroups", typeof(BindingUpdateGroupCollection), typeof(Behavior), new PropertyMetadata(null));
+
+      public static BindingUpdateGroupCollection GetBindingUpdateGroups(DependencyObject d) => d.GetValue(BindingUpdateGroupsProperty) as BindingUpdateGroupCollection;
+      public static void SetBindingUpdateGroups(DependencyObject d, BindingUpdateGroupCollection value) => d.SetValue(BindingUpdateGroupsProperty, value);
+
+      #region AutoRaiseCanExecuteChanged
+      public static readonly DependencyProperty AutoRaiseCanExecuteChangedProperty =
+         DependencyProperty.RegisterAttached("AutoRaiseCanExecuteChanged", typeof(bool), typeof(Behavior),
+            new PropertyMetadata(false, (d, e) => {
+               if ((bool)e.NewValue) {
+                  BindingOperations.SetBinding(d, AutoCommandProperty, new Binding(nameof(ButtonBase.Command)) { Source = d, Mode = BindingMode.OneWay });
+                  BindingOperations.SetBinding(d, AutoCommandParameterProperty, new Binding(nameof(ButtonBase.CommandParameter)) { Source = d, Mode = BindingMode.OneWay });
+               } else {
+                  BindingOperations.ClearBinding(d, AutoCommandProperty);
+                  BindingOperations.ClearBinding(d, AutoCommandParameterProperty);
+               }
+            }));
+
+      public static bool GetAutoRaiseCanExecuteChanged(DependencyObject d) => (bool)d.GetValue(AutoRaiseCanExecuteChangedProperty);
+      public static void SetAutoRaiseCanExecuteChanged(DependencyObject d, bool value) => d.SetValue(AutoRaiseCanExecuteChangedProperty, value);
+
+      /// Do not bind directly to <see cref="AutoCommandProperty"/> (used internally by <see cref="AutoRaiseCanExecuteChangedProperty"/>
+      public static readonly DependencyProperty AutoCommandProperty =
+         DependencyProperty.RegisterAttached("AutoCommand", typeof(ICommand), typeof(Behavior), new PropertyMetadata(null, _try_raise_can_execute));
+
+      public static ICommand GetAutoCommand(DependencyObject d) => d.GetValue(AutoCommandProperty) as ICommand;
+      public static void SetAutoCommand(DependencyObject d, ICommand value) => d.SetValue(AutoCommandProperty, value);
+
+      /// Do not bind directly to <see cref="AutoCommandParameterProperty"/> (used internally by <see cref="AutoRaiseCanExecuteChangedProperty"/>
+      public static readonly DependencyProperty AutoCommandParameterProperty =
+         DependencyProperty.RegisterAttached("AutoCommandParameter", typeof(object), typeof(Behavior), new PropertyMetadata(null, _try_raise_can_execute));
+
+      public static object GetAutoCommandParameter(DependencyObject d) => d.GetValue(AutoCommandParameterProperty);
+      public static void SetAutoCommandParameter(DependencyObject d, object value) => d.SetValue(AutoCommandParameterProperty, value);
+
+      static void _try_raise_can_execute(DependencyObject d, DependencyPropertyChangedEventArgs e) =>
+            (d.GetValue(ButtonBase.CommandProperty) as IRaiseCanExecuteChangedCommand)?.RaiseCanExecuteChanged();
+      #endregion
+
+      #region FocusOnLoaded
+      public static readonly DependencyProperty FocusOnLoadedProperty =
+         DependencyProperty.RegisterAttached("FocusOnLoaded", typeof(bool), typeof(Behavior), new PropertyMetadata(false, OnFocusOnLoadedChanged));
+
+      static void OnFocusOnLoadedChanged(DependencyObject d, DependencyPropertyChangedEventArgs e) {
+         var target = (FrameworkElement)d;
+         if ((bool)e.NewValue) target.Loaded += _target_Loaded;
+         else target.Loaded -= _target_Loaded;
+      }
+
+      static void _target_Loaded(object sender, RoutedEventArgs e) {
+         var target = (FrameworkElement)sender;
+         Keyboard.Focus(target);
+      }
+      public static bool GetFocusOnLoaded(DependencyObject d) => (bool)d.GetValue(FocusOnLoadedProperty);
+      public static void SetFocusOnLoaded(DependencyObject d, bool value) => d.SetValue(FocusOnLoadedProperty, value);
+      #endregion
+
+      #region MirrorLayout
+      static readonly DependencyProperty[] _layout_props = new[] {
+         FrameworkElement.HeightProperty,
+         FrameworkElement.WidthProperty,
+         FrameworkElement.MarginProperty,
+         Control.PaddingProperty,
+         FrameworkElement.HorizontalAlignmentProperty,
+         Control.HorizontalContentAlignmentProperty,
+         FrameworkElement.VerticalAlignmentProperty,
+         Control.VerticalContentAlignmentProperty
+      };
+
+      public static readonly DependencyProperty MirrorLayoutForProperty =
+         DependencyProperty.RegisterAttached("MirrorLayoutFor", typeof(FrameworkElement), typeof(Behavior),
+            new PropertyMetadata(OnMirrorLayoutForChanged));
+
+      static void OnMirrorLayoutForChanged(DependencyObject d, DependencyPropertyChangedEventArgs e) {
+         var target = new LazyCast<FrameworkElement, DependencyObject>(d);
+         if (e.OldValue is FrameworkElement old_src) {
+            target.value.clear_bindings(_layout_props);
+         }
+         if (e.NewValue is FrameworkElement new_src) {
+            target.value.bind(FrameworkElement.WidthProperty, new_src, FrameworkElement.ActualWidthProperty);
+            target.value.bind(FrameworkElement.HeightProperty, new_src, FrameworkElement.ActualHeightProperty);
+            target.value.bind(FrameworkElement.MarginProperty, new_src, FrameworkElement.MarginProperty);
+            target.value.bind(FrameworkElement.HorizontalAlignmentProperty, new_src, FrameworkElement.HorizontalAlignmentProperty);
+            target.value.bind(FrameworkElement.VerticalAlignmentProperty, new_src, FrameworkElement.VerticalAlignmentProperty);
+            if (target.value is Control && new_src is Control) {
+               target.value.bind(Control.PaddingProperty, new_src, Control.PaddingProperty);
+               target.value.bind(Control.HorizontalContentAlignmentProperty, new_src, Control.HorizontalContentAlignmentProperty);
+               target.value.bind(Control.VerticalContentAlignmentProperty, new_src, Control.VerticalContentAlignmentProperty);
+            }
+         }
+      }
+
+      public static FrameworkElement GetMirrorLayoutFor(DependencyObject d) => d.GetValue(MirrorLayoutForProperty) as FrameworkElement;
+      public static void SetMirrorLayoutFor(DependencyObject d, FrameworkElement value) => d.SetValue(MirrorLayoutForProperty, value);
+      #endregion
+   }
+}
@@ -1,348 +1,371 @@
-using bts;
-
-using mbts.ui.native;
-
-using System;
-using System.Runtime.InteropServices;
-using System.Windows;
-using System.Windows.Input;
-using System.Windows.Interop;
-
-namespace mbts.ui {
-   public partial class WindowStyle : ResourceDictionary {
-      // 'Top' is 0 because the window's TitleContent overlaps the top-resize-border.
-      public static readonly Thickness NegativeResizeBorderThickness = new Thickness(-5, 0, -5, -5);
-
-      public static readonly ExecutedRoutedEventHandler
-         CloseWindowExecutedHandler = WindowCallbackHelper.safe_command_executed_handler(w => w.Close(), "closing"),
-         HideWindowExecutedHandler = WindowCallbackHelper.safe_command_executed_handler(w => w.Visibility = Visibility.Hidden, "hiding"),
-         MinimizeWindowExecutedHandler = WindowCallbackHelper.command_executed_handler(w => w.WindowState = WindowState.Minimized),
-         MaximizeWindowExecutedHandler = WindowCallbackHelper.command_executed_handler(w => w.WindowState = w.WindowState == WindowState.Maximized ? WindowState.Normal : WindowState.Maximized);
-
-      #region UnsafeWindow
-      // UnsafeWindow is described as 'Unsafe' because the window may be on a different dispatcher than the DependencyObject target. 
-      public static readonly DependencyProperty UnsafeWindowProperty =
-         DependencyProperty.RegisterAttached("UnsafeWindow", typeof(Window), typeof(WindowStyle), new PropertyMetadata(OnUnsafeWindowChanged));
-
-      static void OnUnsafeWindowChanged(DependencyObject d, DependencyPropertyChangedEventArgs e) {
-         if (d is UIElement el)
-            el.RaiseEvent(new RoutedPropertyChangedEventArgs<Window>(e.OldValue as Window, e.NewValue as Window, UnsafeWindowChangedEvent));
-      }
-
-      public static Window GetUnsafeWindow(DependencyObject d) => d.GetValue(UnsafeWindowProperty) as Window;
-      public static void SetUnsafeWindow(DependencyObject d, Window value) => d.SetValue(UnsafeWindowProperty, value);
-      #endregion
-
-      #region UnsafeWindowChanged (Event)
-      internal static readonly RoutedEvent UnsafeWindowChangedEvent = EventManager.RegisterRoutedEvent("UnsafeWindowChanged", RoutingStrategy.Direct, typeof(RoutedPropertyChangedEventHandler<Window>), typeof(WindowStyle));
-
-      internal static void AddUnsafeWindowChangedHandler(UIElement el, RoutedPropertyChangedEventHandler<Window> handler) =>
-         el.AddHandler(UnsafeWindowChangedEvent, handler);
-
-      internal static void RemoveUnsafeWindowChangedHandler(UIElement el, RoutedPropertyChangedEventHandler<Window> handler) =>
-         el.RemoveHandler(UnsafeWindowChangedEvent, handler);
-      #endregion
-
-      static class WindowCallbackHelper {
-         public static ExecutedRoutedEventHandler safe_command_executed_handler(Action<Window> action, string action_name) =>
-            (s, e) => _safe_execute_on_window(s, action, action_name);
-
-         public static ExecutedRoutedEventHandler command_executed_handler(Action<Window> action) =>
-            (s, e) => _execute_on_window(s, action);
-
-         public static RoutedEventHandler routed_event_handler(Action<Window> action) =>
-            (s, e) => _execute_on_window(s, action);
-
-         static void _execute_on_window(object sender, Action<Window> action) {
-            if (_get_host_window(sender) is Window window)
-               action(window);
-         }
-
-         static void _safe_execute_on_window(object sender, Action<Window> action, string action_name) {
-            try {
-               _execute_on_window(sender, action);
-            } catch (Exception err) {
-               Log.Ui.warn($"Exception {action_name} window : {err}");
-            }
-         }
-
-         static Window _get_host_window(object sender) =>
-            sender is Window w ? w : sender is DependencyObject d ? Window.GetWindow(d) : null;
-      }
-
-      #region Attached properties to control the style b/c the default Window properties are lame
-
-      #region Close button
-      public static bool GetShowCloseButton(DependencyObject obj) {
-         return (bool)obj.GetValue(ShowCloseButtonProperty);
-      }
-      public static void SetShowCloseButton(DependencyObject obj, bool value) {
-         obj.SetValue(ShowCloseButtonProperty, value);
-      }
-      public static readonly DependencyProperty ShowCloseButtonProperty =
-          DependencyProperty.RegisterAttached("ShowCloseButton", typeof(bool), typeof(WindowStyle), new PropertyMetadata(true));
-      #endregion
-
-      #region Minimize Button
-      public static bool GetShowMinimizeButton(DependencyObject obj) {
-         return (bool)obj.GetValue(ShowMinimizeButtonProperty);
-      }
-      public static void SetShowMinimizeButton(DependencyObject obj, bool value) {
-         obj.SetValue(ShowMinimizeButtonProperty, value);
-      }
-      public static readonly DependencyProperty ShowMinimizeButtonProperty =
-          DependencyProperty.RegisterAttached("ShowMinimizeButton", typeof(bool), typeof(WindowStyle), new PropertyMetadata(true));
-      #endregion
-
-      #region Maximize Button
-
-      public static bool GetShowMaximizeButton(DependencyObject obj) {
-         return (bool)obj.GetValue(ShowMaximizeButtonProperty);
-      }
-      public static void SetShowMaximizeButton(DependencyObject obj, bool value) {
-         obj.SetValue(ShowMaximizeButtonProperty, value);
-      }
-      public static readonly DependencyProperty ShowMaximizeButtonProperty =
-          DependencyProperty.RegisterAttached("ShowMaximizeButton", typeof(bool), typeof(WindowStyle), new PropertyMetadata(false));
-      #endregion
-
-      #region AllowResize
-      public static bool GetAllowResize(DependencyObject obj) {
-         return (bool)obj.GetValue(AllowResizeProperty);
-      }
-      public static void SetAllowResize(DependencyObject obj, bool value) {
-         obj.SetValue(AllowResizeProperty, value);
-      }
-      public static readonly DependencyProperty AllowResizeProperty =
-          DependencyProperty.RegisterAttached("AllowResize", typeof(bool), typeof(WindowStyle), new PropertyMetadata(false, (s, e) => {
-             var wnd = s as Window;
-             if (wnd != null) {
-                if ((bool)e.NewValue == true) {
-                   wnd.ResizeMode = ResizeMode.CanResize;
-                } else {
-                   wnd.ResizeMode = ResizeMode.NoResize;
-                }
-             }
-          }));
-      #endregion
-
-      #region IsNonClientArea Inherited property
-
-      public static bool GetIsNonClientArea(DependencyObject obj) {
-         return (bool)obj.GetValue(IsNonClientAreaProperty);
-      }
-
-      public static void SetIsNonClientArea(DependencyObject obj, bool value) {
-         obj.SetValue(IsNonClientAreaProperty, value);
-      }
-
-      public static readonly DependencyProperty IsNonClientAreaProperty =
-          DependencyProperty.RegisterAttached("IsNonClientArea", typeof(bool), typeof(WindowStyle), new FrameworkPropertyMetadata(false, FrameworkPropertyMetadataOptions.Inherits));
-      #endregion
-
-      #region [Custom]SizeToContent property (some additional logic is needed to avoid ugly black border margins on the edge of the window)
-
-      public static readonly DependencyProperty SizeToContentProperty = DependencyProperty.RegisterAttached("SizeToContent", typeof(SizeToContent), typeof(WindowStyle), new PropertyMetadata(SizeToContent.Manual, OnSizeToContentChanged));
-      private static void OnSizeToContentChanged(DependencyObject d, DependencyPropertyChangedEventArgs e) {
-         SizeToContent old_val = (SizeToContent)e.OldValue;
-         SizeToContent new_val = (SizeToContent)e.NewValue;
-         Window w = (Window)d;
-         if (old_val != SizeToContent.Manual) {
-            if (new_val == SizeToContent.Manual) w.ContentRendered -= InvalidateVisualOnContentRendered;
-         } else w.ContentRendered += InvalidateVisualOnContentRendered;
-         w.SizeToContent = new_val;
-      }
-      private static void InvalidateVisualOnContentRendered(object sender, EventArgs e) { ((Window)sender).InvalidateVisual(); }
-      public static SizeToContent GetSizeToContent(Window w) { return (SizeToContent)w.GetValue(SizeToContentProperty); }
-      public static void SetSizeToContent(Window w, SizeToContent value) { w.SetValue(SizeToContentProperty, value); }
-
-      #endregion
-
-      #region TitleContent
-      public static readonly DependencyProperty TitleContentProperty = DependencyProperty.RegisterAttached("TitleContent", typeof(object), typeof(WindowStyle), new PropertyMetadata(null));
-      public static object GetTitleContent(DependencyObject d) { return d.GetValue(TitleContentProperty); }
-      public static void SetTitleContent(DependencyObject d, object value) { d.SetValue(TitleContentProperty, value); }
-      #endregion
-
-      #region TitleContentTemplate
-      public static readonly DependencyProperty TitleContentTemplateProperty = DependencyProperty.RegisterAttached("TitleContentTemplate", typeof(DataTemplate), typeof(WindowStyle), new PropertyMetadata(null));
-      public static DataTemplate GetTitleContentTemplate(DependencyObject d) { return d.GetValue(TitleContentTemplateProperty) as DataTemplate; }
-      public static void SetTitleContentTemplate(DependencyObject d, DataTemplate value) { d.SetValue(TitleContentTemplateProperty, value); }
-      #endregion
-
-      #region DisposeVMOnClose
-      public static readonly DependencyProperty DisposeVMOnCloseProperty = DependencyProperty.RegisterAttached("DisposeVMOnClose", typeof(bool), typeof(WindowStyle), new PropertyMetadata(false, OnDisposeVMOnCloseChanged));
-      private static void OnDisposeVMOnCloseChanged(DependencyObject d, DependencyPropertyChangedEventArgs e) {
-         var target = d as Window;
-         target?.SetValue(DisposeVMOnCloseHandlerWrapperPropertyKey, (bool)e.NewValue ? new DisposeVMOnCloseHandlerWrapper(target) : null);
-      }
-      public static bool GetDisposeVMOnClose(DependencyObject d) { return (bool)d.GetValue(DisposeVMOnCloseProperty); }
-      public static void SetDisposeVMOnClose(DependencyObject d, bool value) { d.SetValue(DisposeVMOnCloseProperty, value); }
-
-      private static readonly DependencyPropertyKey DisposeVMOnCloseHandlerWrapperPropertyKey = DependencyProperty.RegisterAttachedReadOnly("DisposeVMOnCloseHandlerWrapper", typeof(IDisposable), typeof(WindowStyle), new PropertyMetadata(null, FrameworkElementLifeDependencyHelper.DisposeOld));
-      internal static readonly DependencyProperty DisposeVMOnCloseHandlerWrapperProperty = DisposeVMOnCloseHandlerWrapperPropertyKey.DependencyProperty;
-      internal static IDisposable GetDisposeVMOnCloseHandlerWrapper(Window w) { return w.GetValue(DisposeVMOnCloseHandlerWrapperProperty) as IDisposable; }
-      private static void SetDisposeVMOnCloseHandlerWrapper(Window w, IDisposable value) { w.SetValue(DisposeVMOnCloseHandlerWrapperProperty, value); }
-
-      private class DisposeVMOnCloseHandlerWrapper : IDisposable {
-         public DisposeVMOnCloseHandlerWrapper(Window target) { (_target = target).Closed += _target_Closed; }
-         private readonly Window _target;
-         private void _target_Closed(object sender, EventArgs e) { (_target.DataContext as IDisposable)?.Dispose(); }
-         public void Dispose() { _target.Closed -= _target_Closed; }
-      }
-      #endregion
-
-      #endregion
-
-      private void style_loaded(object sender, RoutedEventArgs _) {
-         if (sender is DependencyObject) {
-            var window = Window.GetWindow(((DependencyObject)sender));
-            if (window != null) {
-               if (WindowStyle.GetAllowResize(window)) {
-                  set_resize_window_events(sender as FrameworkElement, window);
-               } else {
-                  // Simpler dragmove interaction
-                  var border = (FrameworkElement)sender;
-                  border.MouseLeftButtonDown += (s, e) => {
-                     if (Mouse.LeftButton == MouseButtonState.Pressed) {
-                        window.DragMove();
-                     }
-                  };
-               }
-
-               // Handle Close command automagically
-               if (window.CommandBindings.Count == 0) {
-                  window.CommandBindings.Add(new CommandBinding(SystemCommands.CloseWindowCommand, CloseWindowExecutedHandler));
-                  window.CommandBindings.Add(new CommandBinding(SystemCommands.MinimizeWindowCommand, MinimizeWindowExecutedHandler));
-                  window.CommandBindings.Add(new CommandBinding(SystemCommands.MaximizeWindowCommand, MaximizeWindowExecutedHandler));
-               }
-            }
-         }
-      }
-
-      private static void set_resize_window_events(FrameworkElement border, Window window) {
-         HookWndProc(window);
-         Log.assert(border != null);
-         var restore_on_move = false;
-         border.MouseLeftButtonDown += (s, e) => {
-            FrameworkElement fe = e.OriginalSource as FrameworkElement;
-            if (fe != null && (bool)fe.GetValue(WindowStyle.IsNonClientAreaProperty)) {
-               if (e.ClickCount == 2) {
-                  toggle_max(window);
-               } else {
-                  if (window.WindowState == WindowState.Maximized) { restore_on_move = true; }
-                  if (Mouse.LeftButton == MouseButtonState.Pressed) {
-                     window.DragMove();
-                  }
-               }
-            }
-         };
-         border.MouseLeftButtonUp += (s, e) => restore_on_move = false;
-         border.MouseMove += (s, e) => {
-            if (restore_on_move && window.WindowState == WindowState.Maximized && Mouse.LeftButton == MouseButtonState.Pressed) {
-               window.WindowState = WindowState.Normal;
-               window.Top = 0;
-               if (Mouse.LeftButton == MouseButtonState.Pressed) {
-                  window.DragMove();
-               }
-            }
-         };
-      }
-
-      private static void toggle_max(Window window) {
-         if (window.WindowState == WindowState.Maximized) {
-            window.WindowState = WindowState.Normal;
-         } else {
-            window.WindowState = WindowState.Maximized;
-         }
-      }
-
-
-
-      private static void HookWndProc(Window window) {
-         HwndSource source = PresentationSource.FromVisual(window) as HwndSource;
-         if (source != null) {
-            source.AddHook(new HwndSourceHook(WndProc));
-         }
-      }
-      private static void UnhookWndProc(Window window) {
-         HwndSource source = PresentationSource.FromVisual(window) as HwndSource;
-         if (source != null) {
-            source.RemoveHook(new HwndSourceHook(WndProc));
-         }
-      }
-
-      private static IntPtr WndProc(IntPtr hwnd, int msg, IntPtr wparam, IntPtr lparam, ref bool handled) {
-         WM wmsg = (WM)msg;
-         if (WM.NCHITTEST == wmsg /*|| WM.MOUSEMOVE == wmsg*/) {
-            Thickness t = new Thickness(6);
-            // Get the mouse coordinates in screen space
-            short x = (short)lparam.ToInt32();
-            short y = (short)(lparam.ToInt32() >> 16);
-            // Get the window bounds in screen coords
-            Win32Rect w;
-            NativeMethods.GetWindowRect(hwnd, out w);
-            // Assume that mouse position is inside the window
-            bool left = x < (w.Left + t.Left);
-            bool right = x > (w.Right - t.Right);
-            bool top = y < (w.Top + t.Top);
-            bool bottom = y > (w.Bottom - t.Bottom);
-
-            //  A  B  C
-            //  D     E
-            //  F  G  H
-
-            HT ret = HT.CLIENT;
-            if (top) {
-               if (left) ret = HT.TOPLEFT; // A
-               else if (right) ret = HT.TOPRIGHT; // C
-               else ret = HT.TOP; // B
-            } else if (bottom) {
-               if (left) ret = HT.BOTTOMLEFT; // F
-               else if (right) ret = HT.BOTTOMRIGHT; // H
-               else ret = HT.BOTTOM; // G
-            } else {
-               if (left) ret = HT.LEFT; // D
-               if (right) ret = HT.RIGHT; // E
-            }
-
-            handled = true;
-            return new IntPtr((int)ret);
-         } else if (wmsg == WM.NCCALCSIZE) {
-            handled = true;
-         } else if (wmsg == WM.NCPAINT) {
-            handled = true;
-         } else if (wmsg == WM.NCACTIVATE) {
-            handled = true;
-            return NativeMethods.DefWindowProc(hwnd, msg, wparam, new IntPtr(-1));
-         } else if (wmsg == WM.ERASEBKGND) {
-            handled = true;
-         } else if (wmsg == WM.GETMINMAXINFO) {
-            SetMinMaxInfo(hwnd, lparam);
-            handled = true;
-         }
-
-         return IntPtr.Zero;
-      }
-
-      private static void SetMinMaxInfo(IntPtr hwnd, IntPtr lparam) {
-         var mminfo = (MINMAXINFO)Marshal.PtrToStructure(lparam, typeof(MINMAXINFO));
-
-         var hmonitor = NativeMethods.MonitorFromWindow(hwnd, NativeMethods.MONITOR_DEFAULTONNEAREST);
-         if (hmonitor != IntPtr.Zero) {
-            MONITORINFO minfo = new MONITORINFO();
-            minfo.cbSize = Marshal.SizeOf(minfo);
-            NativeMethods.GetMonitorInfo(hmonitor, ref minfo);
-            var rc_monitor = minfo.rcMonitor;
-            var rc_work = minfo.rcWork;
-            mminfo.ptMaxPosition.x = Math.Abs(rc_work.Left - rc_monitor.Left);
-            mminfo.ptMaxPosition.y = Math.Abs(rc_work.Top - rc_monitor.Top);
-            mminfo.ptMaxSize.x = Math.Abs(rc_work.Right - rc_work.Left);
-            mminfo.ptMaxSize.y = Math.Abs(rc_work.Bottom - rc_work.Top);
-
-            Marshal.StructureToPtr(mminfo, lparam, true);
-         }
-      }
-
-   }
-}
+using bts;
+
+using mbts.ui.native;
+
+using System;
+using System.Runtime.InteropServices;
+using System.Windows;
+using System.Windows.Input;
+using System.Windows.Interop;
+
+namespace mbts.ui {
+   public partial class WindowStyle : ResourceDictionary {
+      // 'Top' is 0 because the window's TitleContent overlaps the top-resize-border.
+      public static readonly Thickness NegativeResizeBorderThickness = new Thickness(-5, 0, -5, -5);
+
+      public static readonly ExecutedRoutedEventHandler
+         CloseWindowExecutedHandler = WindowCallbackHelper.safe_command_executed_handler(w => w.Close(), "closing"),
+         HideWindowExecutedHandler = WindowCallbackHelper.safe_command_executed_handler(w => w.Visibility = Visibility.Hidden, "hiding"),
+         MinimizeWindowExecutedHandler = WindowCallbackHelper.command_executed_handler(w => w.WindowState = WindowState.Minimized),
+         MaximizeWindowExecutedHandler = WindowCallbackHelper.command_executed_handler(w => w.WindowState = w.WindowState == WindowState.Maximized ? WindowState.Normal : WindowState.Maximized);
+
+      #region UnsafeWindow
+      // UnsafeWindow is described as 'Unsafe' because the window may be on a different dispatcher than the DependencyObject target. 
+      public static readonly DependencyProperty UnsafeWindowProperty =
+         DependencyProperty.RegisterAttached("UnsafeWindow", typeof(Window), typeof(WindowStyle), new PropertyMetadata(OnUnsafeWindowChanged));
+
+      static void OnUnsafeWindowChanged(DependencyObject d, DependencyPropertyChangedEventArgs e) {
+         if (d is UIElement el)
+            el.RaiseEvent(new RoutedPropertyChangedEventArgs<Window>(e.OldValue as Window, e.NewValue as Window, UnsafeWindowChangedEvent));
+      }
+
+      public static Window GetUnsafeWindow(DependencyObject d) => d.GetValue(UnsafeWindowProperty) as Window;
+      public static void SetUnsafeWindow(DependencyObject d, Window value) => d.SetValue(UnsafeWindowProperty, value);
+      #endregion
+
+      #region UnsafeWindowChanged (Event)
+      internal static readonly RoutedEvent UnsafeWindowChangedEvent = EventManager.RegisterRoutedEvent("UnsafeWindowChanged", RoutingStrategy.Direct, typeof(RoutedPropertyChangedEventHandler<Window>), typeof(WindowStyle));
+
+      internal static void AddUnsafeWindowChangedHandler(UIElement el, RoutedPropertyChangedEventHandler<Window> handler) =>
+         el.AddHandler(UnsafeWindowChangedEvent, handler);
+
+      internal static void RemoveUnsafeWindowChangedHandler(UIElement el, RoutedPropertyChangedEventHandler<Window> handler) =>
+         el.RemoveHandler(UnsafeWindowChangedEvent, handler);
+      #endregion
+
+      #region WinFormsHost
+      public static readonly DependencyProperty WinFormsHostProperty =
+         DependencyProperty.RegisterAttached("WinFormsHost", typeof(WPFPanel), typeof(WindowStyle), new PropertyMetadata(OnWinFormsHostChanged));
+
+      static void OnWinFormsHostChanged(DependencyObject d, DependencyPropertyChangedEventArgs e) {
+         if (d is UIElement el)
+            el.RaiseEvent(new RoutedPropertyChangedEventArgs<WPFPanel>(e.OldValue as WPFPanel, e.NewValue as WPFPanel, WinFormsHostChangedEvent));
+      }
+
+      public static WPFPanel GetWinFormsHost(DependencyObject d) => d.GetValue(WinFormsHostProperty) as WPFPanel;
+      public static void SetWinFormsHost(DependencyObject d, WPFPanel value) => d.SetValue(WinFormsHostProperty, value);
+      #endregion
+
+      #region WinFormsHostChanged (Event)
+      internal static readonly RoutedEvent WinFormsHostChangedEvent = EventManager.RegisterRoutedEvent("WinFormsHostChanged", RoutingStrategy.Direct, typeof(RoutedPropertyChangedEventHandler<WPFPanel>), typeof(WindowStyle));
+
+      internal static void AddWinFormsHostChangedHandler(UIElement el, RoutedPropertyChangedEventHandler<WPFPanel> handler) =>
+         el.AddHandler(WinFormsHostChangedEvent, handler);
+
+      internal static void RemoveWinFormsHostChangedHandler(UIElement el, RoutedPropertyChangedEventHandler<WPFPanel> handler) =>
+         el.RemoveHandler(WinFormsHostChangedEvent, handler);
+      #endregion
+
+      static class WindowCallbackHelper {
+         public static ExecutedRoutedEventHandler safe_command_executed_handler(Action<Window> action, string action_name) =>
+            (s, e) => _safe_execute_on_window(s, action, action_name);
+
+         public static ExecutedRoutedEventHandler command_executed_handler(Action<Window> action) =>
+            (s, e) => _execute_on_window(s, action);
+
+         public static RoutedEventHandler routed_event_handler(Action<Window> action) =>
+            (s, e) => _execute_on_window(s, action);
+
+         static void _execute_on_window(object sender, Action<Window> action) {
+            if (_get_host_window(sender) is Window window)
+               action(window);
+         }
+
+         static void _safe_execute_on_window(object sender, Action<Window> action, string action_name) {
+            try {
+               _execute_on_window(sender, action);
+            } catch (Exception err) {
+               Log.Ui.warn($"Exception {action_name} window : {err}");
+            }
+         }
+
+         static Window _get_host_window(object sender) =>
+            sender is Window w ? w : sender is DependencyObject d ? Window.GetWindow(d) : null;
+      }
+
+      #region Attached properties to control the style b/c the default Window properties are lame
+
+      #region Close button
+      public static bool GetShowCloseButton(DependencyObject obj) {
+         return (bool)obj.GetValue(ShowCloseButtonProperty);
+      }
+      public static void SetShowCloseButton(DependencyObject obj, bool value) {
+         obj.SetValue(ShowCloseButtonProperty, value);
+      }
+      public static readonly DependencyProperty ShowCloseButtonProperty =
+          DependencyProperty.RegisterAttached("ShowCloseButton", typeof(bool), typeof(WindowStyle), new PropertyMetadata(true));
+      #endregion
+
+      #region Minimize Button
+      public static bool GetShowMinimizeButton(DependencyObject obj) {
+         return (bool)obj.GetValue(ShowMinimizeButtonProperty);
+      }
+      public static void SetShowMinimizeButton(DependencyObject obj, bool value) {
+         obj.SetValue(ShowMinimizeButtonProperty, value);
+      }
+      public static readonly DependencyProperty ShowMinimizeButtonProperty =
+          DependencyProperty.RegisterAttached("ShowMinimizeButton", typeof(bool), typeof(WindowStyle), new PropertyMetadata(true));
+      #endregion
+
+      #region Maximize Button
+
+      public static bool GetShowMaximizeButton(DependencyObject obj) {
+         return (bool)obj.GetValue(ShowMaximizeButtonProperty);
+      }
+      public static void SetShowMaximizeButton(DependencyObject obj, bool value) {
+         obj.SetValue(ShowMaximizeButtonProperty, value);
+      }
+      public static readonly DependencyProperty ShowMaximizeButtonProperty =
+          DependencyProperty.RegisterAttached("ShowMaximizeButton", typeof(bool), typeof(WindowStyle), new PropertyMetadata(false));
+      #endregion
+
+      #region AllowResize
+      public static bool GetAllowResize(DependencyObject obj) {
+         return (bool)obj.GetValue(AllowResizeProperty);
+      }
+      public static void SetAllowResize(DependencyObject obj, bool value) {
+         obj.SetValue(AllowResizeProperty, value);
+      }
+      public static readonly DependencyProperty AllowResizeProperty =
+          DependencyProperty.RegisterAttached("AllowResize", typeof(bool), typeof(WindowStyle), new PropertyMetadata(false, (s, e) => {
+             var wnd = s as Window;
+             if (wnd != null) {
+                if ((bool)e.NewValue == true) {
+                   wnd.ResizeMode = ResizeMode.CanResize;
+                } else {
+                   wnd.ResizeMode = ResizeMode.NoResize;
+                }
+             }
+          }));
+      #endregion
+
+      #region IsNonClientArea Inherited property
+
+      public static bool GetIsNonClientArea(DependencyObject obj) {
+         return (bool)obj.GetValue(IsNonClientAreaProperty);
+      }
+
+      public static void SetIsNonClientArea(DependencyObject obj, bool value) {
+         obj.SetValue(IsNonClientAreaProperty, value);
+      }
+
+      public static readonly DependencyProperty IsNonClientAreaProperty =
+          DependencyProperty.RegisterAttached("IsNonClientArea", typeof(bool), typeof(WindowStyle), new FrameworkPropertyMetadata(false, FrameworkPropertyMetadataOptions.Inherits));
+      #endregion
+
+      #region [Custom]SizeToContent property (some additional logic is needed to avoid ugly black border margins on the edge of the window)
+
+      public static readonly DependencyProperty SizeToContentProperty = DependencyProperty.RegisterAttached("SizeToContent", typeof(SizeToContent), typeof(WindowStyle), new PropertyMetadata(SizeToContent.Manual, OnSizeToContentChanged));
+      private static void OnSizeToContentChanged(DependencyObject d, DependencyPropertyChangedEventArgs e) {
+         SizeToContent old_val = (SizeToContent)e.OldValue;
+         SizeToContent new_val = (SizeToContent)e.NewValue;
+         Window w = (Window)d;
+         if (old_val != SizeToContent.Manual) {
+            if (new_val == SizeToContent.Manual) w.ContentRendered -= InvalidateVisualOnContentRendered;
+         } else w.ContentRendered += InvalidateVisualOnContentRendered;
+         w.SizeToContent = new_val;
+      }
+      private static void InvalidateVisualOnContentRendered(object sender, EventArgs e) { ((Window)sender).InvalidateVisual(); }
+      public static SizeToContent GetSizeToContent(Window w) { return (SizeToContent)w.GetValue(SizeToContentProperty); }
+      public static void SetSizeToContent(Window w, SizeToContent value) { w.SetValue(SizeToContentProperty, value); }
+
+      #endregion
+
+      #region TitleContent
+      public static readonly DependencyProperty TitleContentProperty = DependencyProperty.RegisterAttached("TitleContent", typeof(object), typeof(WindowStyle), new PropertyMetadata(null));
+      public static object GetTitleContent(DependencyObject d) { return d.GetValue(TitleContentProperty); }
+      public static void SetTitleContent(DependencyObject d, object value) { d.SetValue(TitleContentProperty, value); }
+      #endregion
+
+      #region TitleContentTemplate
+      public static readonly DependencyProperty TitleContentTemplateProperty = DependencyProperty.RegisterAttached("TitleContentTemplate", typeof(DataTemplate), typeof(WindowStyle), new PropertyMetadata(null));
+      public static DataTemplate GetTitleContentTemplate(DependencyObject d) { return d.GetValue(TitleContentTemplateProperty) as DataTemplate; }
+      public static void SetTitleContentTemplate(DependencyObject d, DataTemplate value) { d.SetValue(TitleContentTemplateProperty, value); }
+      #endregion
+
+      #region DisposeVMOnClose
+      public static readonly DependencyProperty DisposeVMOnCloseProperty = DependencyProperty.RegisterAttached("DisposeVMOnClose", typeof(bool), typeof(WindowStyle), new PropertyMetadata(false, OnDisposeVMOnCloseChanged));
+      private static void OnDisposeVMOnCloseChanged(DependencyObject d, DependencyPropertyChangedEventArgs e) {
+         var target = d as Window;
+         target?.SetValue(DisposeVMOnCloseHandlerWrapperPropertyKey, (bool)e.NewValue ? new DisposeVMOnCloseHandlerWrapper(target) : null);
+      }
+      public static bool GetDisposeVMOnClose(DependencyObject d) { return (bool)d.GetValue(DisposeVMOnCloseProperty); }
+      public static void SetDisposeVMOnClose(DependencyObject d, bool value) { d.SetValue(DisposeVMOnCloseProperty, value); }
+
+      private static readonly DependencyPropertyKey DisposeVMOnCloseHandlerWrapperPropertyKey = DependencyProperty.RegisterAttachedReadOnly("DisposeVMOnCloseHandlerWrapper", typeof(IDisposable), typeof(WindowStyle), new PropertyMetadata(null, FrameworkElementLifeDependencyHelper.DisposeOld));
+      internal static readonly DependencyProperty DisposeVMOnCloseHandlerWrapperProperty = DisposeVMOnCloseHandlerWrapperPropertyKey.DependencyProperty;
+      internal static IDisposable GetDisposeVMOnCloseHandlerWrapper(Window w) { return w.GetValue(DisposeVMOnCloseHandlerWrapperProperty) as IDisposable; }
+      private static void SetDisposeVMOnCloseHandlerWrapper(Window w, IDisposable value) { w.SetValue(DisposeVMOnCloseHandlerWrapperProperty, value); }
+
+      private class DisposeVMOnCloseHandlerWrapper : IDisposable {
+         public DisposeVMOnCloseHandlerWrapper(Window target) { (_target = target).Closed += _target_Closed; }
+         private readonly Window _target;
+         private void _target_Closed(object sender, EventArgs e) { (_target.DataContext as IDisposable)?.Dispose(); }
+         public void Dispose() { _target.Closed -= _target_Closed; }
+      }
+      #endregion
+
+      #endregion
+
+      private void style_loaded(object sender, RoutedEventArgs _) {
+         if (sender is DependencyObject) {
+            var window = Window.GetWindow(((DependencyObject)sender));
+            if (window != null) {
+               if (WindowStyle.GetAllowResize(window)) {
+                  set_resize_window_events(sender as FrameworkElement, window);
+               } else {
+                  // Simpler dragmove interaction
+                  var border = (FrameworkElement)sender;
+                  border.MouseLeftButtonDown += (s, e) => {
+                     if (Mouse.LeftButton == MouseButtonState.Pressed) {
+                        window.DragMove();
+                     }
+                  };
+               }
+
+               // Handle Close command automagically
+               if (window.CommandBindings.Count == 0) {
+                  window.CommandBindings.Add(new CommandBinding(SystemCommands.CloseWindowCommand, CloseWindowExecutedHandler));
+                  window.CommandBindings.Add(new CommandBinding(SystemCommands.MinimizeWindowCommand, MinimizeWindowExecutedHandler));
+                  window.CommandBindings.Add(new CommandBinding(SystemCommands.MaximizeWindowCommand, MaximizeWindowExecutedHandler));
+               }
+            }
+         }
+      }
+
+      private static void set_resize_window_events(FrameworkElement border, Window window) {
+         HookWndProc(window);
+         Log.assert(border != null);
+         var restore_on_move = false;
+         border.MouseLeftButtonDown += (s, e) => {
+            FrameworkElement fe = e.OriginalSource as FrameworkElement;
+            if (fe != null && (bool)fe.GetValue(WindowStyle.IsNonClientAreaProperty)) {
+               if (e.ClickCount == 2) {
+                  toggle_max(window);
+               } else {
+                  if (window.WindowState == WindowState.Maximized) { restore_on_move = true; }
+                  if (Mouse.LeftButton == MouseButtonState.Pressed) {
+                     window.DragMove();
+                  }
+               }
+            }
+         };
+         border.MouseLeftButtonUp += (s, e) => restore_on_move = false;
+         border.MouseMove += (s, e) => {
+            if (restore_on_move && window.WindowState == WindowState.Maximized && Mouse.LeftButton == MouseButtonState.Pressed) {
+               window.WindowState = WindowState.Normal;
+               window.Top = 0;
+               if (Mouse.LeftButton == MouseButtonState.Pressed) {
+                  window.DragMove();
+               }
+            }
+         };
+      }
+
+      private static void toggle_max(Window window) {
+         if (window.WindowState == WindowState.Maximized) {
+            window.WindowState = WindowState.Normal;
+         } else {
+            window.WindowState = WindowState.Maximized;
+         }
+      }
+
+
+
+      private static void HookWndProc(Window window) {
+         HwndSource source = PresentationSource.FromVisual(window) as HwndSource;
+         if (source != null) {
+            source.AddHook(new HwndSourceHook(WndProc));
+         }
+      }
+      private static void UnhookWndProc(Window window) {
+         HwndSource source = PresentationSource.FromVisual(window) as HwndSource;
+         if (source != null) {
+            source.RemoveHook(new HwndSourceHook(WndProc));
+         }
+      }
+
+      private static IntPtr WndProc(IntPtr hwnd, int msg, IntPtr wparam, IntPtr lparam, ref bool handled) {
+         WM wmsg = (WM)msg;
+         if (WM.NCHITTEST == wmsg /*|| WM.MOUSEMOVE == wmsg*/) {
+            Thickness t = new Thickness(6);
+            // Get the mouse coordinates in screen space
+            short x = (short)lparam.ToInt32();
+            short y = (short)(lparam.ToInt32() >> 16);
+            // Get the window bounds in screen coords
+            Win32Rect w;
+            NativeMethods.GetWindowRect(hwnd, out w);
+            // Assume that mouse position is inside the window
+            bool left = x < (w.Left + t.Left);
+            bool right = x > (w.Right - t.Right);
+            bool top = y < (w.Top + t.Top);
+            bool bottom = y > (w.Bottom - t.Bottom);
+
+            //  A  B  C
+            //  D     E
+            //  F  G  H
+
+            HT ret = HT.CLIENT;
+            if (top) {
+               if (left) ret = HT.TOPLEFT; // A
+               else if (right) ret = HT.TOPRIGHT; // C
+               else ret = HT.TOP; // B
+            } else if (bottom) {
+               if (left) ret = HT.BOTTOMLEFT; // F
+               else if (right) ret = HT.BOTTOMRIGHT; // H
+               else ret = HT.BOTTOM; // G
+            } else {
+               if (left) ret = HT.LEFT; // D
+               if (right) ret = HT.RIGHT; // E
+            }
+
+            handled = true;
+            return new IntPtr((int)ret);
+         } else if (wmsg == WM.NCCALCSIZE) {
+            handled = true;
+         } else if (wmsg == WM.NCPAINT) {
+            handled = true;
+         } else if (wmsg == WM.NCACTIVATE) {
+            handled = true;
+            return NativeMethods.DefWindowProc(hwnd, msg, wparam, new IntPtr(-1));
+         } else if (wmsg == WM.ERASEBKGND) {
+            handled = true;
+         } else if (wmsg == WM.GETMINMAXINFO) {
+            SetMinMaxInfo(hwnd, lparam);
+            handled = true;
+         }
+
+         return IntPtr.Zero;
+      }
+
+      private static void SetMinMaxInfo(IntPtr hwnd, IntPtr lparam) {
+         var mminfo = (MINMAXINFO)Marshal.PtrToStructure(lparam, typeof(MINMAXINFO));
+
+         var hmonitor = NativeMethods.MonitorFromWindow(hwnd, NativeMethods.MONITOR_DEFAULTONNEAREST);
+         if (hmonitor != IntPtr.Zero) {
+            MONITORINFO minfo = new MONITORINFO();
+            minfo.cbSize = Marshal.SizeOf(minfo);
+            NativeMethods.GetMonitorInfo(hmonitor, ref minfo);
+            var rc_monitor = minfo.rcMonitor;
+            var rc_work = minfo.rcWork;
+            mminfo.ptMaxPosition.x = Math.Abs(rc_work.Left - rc_monitor.Left);
+            mminfo.ptMaxPosition.y = Math.Abs(rc_work.Top - rc_monitor.Top);
+            mminfo.ptMaxSize.x = Math.Abs(rc_work.Right - rc_work.Left);
+            mminfo.ptMaxSize.y = Math.Abs(rc_work.Bottom - rc_work.Top);
+
+            Marshal.StructureToPtr(mminfo, lparam, true);
+         }
+      }
+
+   }
+}
@@ -1,241 +1,242 @@
-namespace mbts.ui.shell {
-   using bts;
-
-   using Spark.Core;
-   using Spark.Core.Async;
-   using Spark.Core.Disposables;
-
-   using System;
-   using System.Reactive.Linq;
-   using System.Runtime.InteropServices;
-   using System.Windows;
-   using System.Windows.Controls;
-   using System.Windows.Input;
-   using System.Windows.Media;
-   using System.Windows.Threading;
-
-   public class ShellWindowVM : DisposableViewModelBase {
-      readonly IWindowManager _window_manager;
-      readonly Property<bool> _edit_mode;
-      readonly Property<string> _title;
-      readonly Property<string> _title_override;
-      readonly Property<bool> _has_custom_title;
-
-      IShellWindow _model;
-      UIElement _ui;
-      ShellWindowUI _document;
-      HwndContentHost _hwnd_host;
-      FloatingWindow _owner;
-      ImageSource _icon;
-      ICommand _close_command;
-      ICommand _rename_command;
-      ICommand _unrename_command;
-
-      public ShellWindowVM(IShellWindow model, IWindowManager wm, IShellThreadFactory threads, string host = null) {
-         Log.assert(model != null);
-         _model = model;
-         _model.IconChanged += on_icon_changed;
-         _window_manager = wm;
-         host_id = (model as ITabAffineWindow)?.tab_group ?? host ?? string.Empty;
-
-         // init Dispatcher
-         var stf = (ShellThreadFactory)threads;
-         if (IsShared) {
-            Dispatcher = stf.get_dispatcher((ISharedWindow)model);
-         } else if (IsSlow) {
-            Dispatcher = stf.create_dispatcher((ISlowWindow)model);
-         } else {
-            Dispatcher = Dispatcher.CurrentDispatcher;
-         }
-
-         _edit_mode = backing(nameof(EditMode), false);
-
-         _title_override = backing<string>(nameof(TitleOverride));
-         _title = backing(
-            nameof(Title),
-            _model.observe_title()
-            .CombineLatest(
-               _title_override.observe_current(true),
-               (title, title_override) => (title, title_override))
-            .Select(t => t.title_override ?? t.title)
-            .OnDispatcher());
-         _has_custom_title = backing(
-            nameof(HasCustomTitle),
-            _title_override.observe_current(true).Select(t => !t.is_null_or_white_space()).OnDispatcher());
-
-         model.on_shell_window_vm(this);
-
-         _dispose = Disposable.Create(() => {
-            _model.IconChanged -= on_icon_changed;
-            if (IsShared) {
-               // Shared threads continue running
-               Dispatcher.Invoke(() => Disposable.dispose(ref _model));
-            } else if (IsSlow) {
-               // Terminating the background thread will dispose the model on that thread
-               Log.Ui.info("Beginning asyncronous dispose of slow window {0} .".fmt(_model.GetType().ToString()));
-               Dispatcher.BeginInvokeShutdown(DispatcherPriority.Background);
-            } else {
-               Disposable.dispose(ref _model);
-            }
-            _ui = null;
-            Disposable.dispose(ref _hwnd_host);
-         });
-
-         void on_icon_changed() {
-            _icon = null;
-            RaisePropertyChanged(nameof(Icon));
-         }
-      }
-
-      public AsyncEvent<(object sender, ShellWindowVM e)> Closing { get; } = new();
-
-      public string host_id { get; set; }
-      public string OwnerName => _model.Owner?.name ?? string.Empty;
-      public IShellWindow Model => _model;
-      public Dispatcher Dispatcher { get; }
-
-      // title
-      public string Title => _title.Value;
-      public string TitleOverride {
-         get => _title_override.Value;
-         set {
-            _title_override.Value = value;
-            _edit_mode.set(false);
-         }
-      }
-      public bool HasCustomTitle => _has_custom_title.Value;
-
-      public ImageSource Icon =>
-         _icon ??= _model.Icon ?? _model.Owner?.icon;
-
-      public bool EditMode => _edit_mode.Value;
-      public double InitialWidth { get; private set; }
-      public double InitialHeight { get; private set; }
-
-      public UIElement AppContent =>
-         _ui ??= _create_contents();
-
-      public bool IsSlow => _model is ISlowWindow;
-      public bool IsShared => _model is ISharedWindow;
-      public bool HasTabAffinity => _model is ITabAffineWindow;
-
-      public ICommand rename_command =>
-         _rename_command ??= new DelegateCommand(_begin_rename);
-
-      public ICommand unrename_command =>
-         _unrename_command ??= new DelegateCommand(() => TitleOverride = string.Empty);
-
-      public ICommand CloseCommand =>
-         _close_command ??= new DelegateCommand(_begin_close);
-
-      internal FloatingWindow owner {
-         get => _owner;
-         set {
-            if (Set(ref _owner, value)) {
-               _try_set_wpf_bus();
-            }
-         }
-      }
-
-      public IObservable<string> observe_title() =>
-         _title.observe_current(true);
-
-      void _begin_rename() {
-         if (string.IsNullOrEmpty(TitleOverride)) {
-            _title_override.Value = Title;
-         }
-         _edit_mode.set(true);
-      }
-
-      async void _begin_close(object sender) {
-         if (_window_manager is null) {
-            return;
-         }
-         if (!_window_manager.contains(_model)) {
-            return;
-         }
-         await Closing?.InvokeAsync((sender, this));
-         // Removing the window will cause this object to be Disposed
-         // Which will terminate the background thread
-         // Which will dispose the model (on that thread)
-         await _window_manager.remove(_model);
-      }
-
-      UIElement _create_contents() {
-         string error_text = null;
-
-         Dispatcher.Invoke(() => {
-            // Executes possibly on background thread
-            try {
-               _document = _model.GetContent();
-            } catch (Exception ex) {
-               error_text = ex.Message;
-               Log.Ui.error($"Error getting window contents: {ex}");
-            }
-         }, DispatcherPriority.Send);
-
-         if (_document == null && string.IsNullOrEmpty(error_text)) {
-            error_text = $"No content for window {_model.GetType()}";
-         }
-
-         // If there's an error draw a big red X
-         if (!error_text.is_null_or_empty()) {
-            _document = new ShellWindowUI(new TextBlock() {
-               HorizontalAlignment = HorizontalAlignment.Center,
-               VerticalAlignment = VerticalAlignment.Center,
-               Foreground = Brushes.Red,
-               Text = error_text
-            });
-            _model = new ErrorShellWindow {
-               Content = _document,
-               Icon = _model.Icon,
-               Title = $"Error: {_model.GetType()}"
-            };
-            RaisePropertyChanged("Title");
-         }
-
-         Log.assert(_document != null);
-         return _document.content.Match(on_wpf_element, on_handle_ref);
-
-         UIElement on_wpf_element(UIElement element) {
-            if (IsSlow || IsShared) {
-               // Create a WPFPanel (a Winforms window), then attach it with the HwndContentHost.
-               // This gives the document content a separate thread affinity from the shell window,
-               // and lets the win32 layer handle the thread boundary.
-               var retval = Dispatcher.Invoke(() => {
-                  var interop_window = new WPFPanel(element);
-                  return new HandleRef(interop_window, interop_window.Handle);
-               }, DispatcherPriority.Send);
-               _hwnd_host = new HwndContentHost(retval);
-               InitialHeight = _hwnd_host.InitialHeight;
-               InitialWidth = _hwnd_host.InitialWidth;
-               return _hwnd_host;
-            }
-            if (element is FrameworkElement fe) {
-               InitialHeight = fe.Height;
-               InitialWidth = fe.Width;
-               fe.Height = fe.Width = double.NaN;
-            }
-            return element;
-         }
-
-         UIElement on_handle_ref(HandleRef handle_ref) {
-            _hwnd_host = new HwndContentHost(handle_ref);
-            InitialHeight = _hwnd_host.Height;
-            InitialWidth = _hwnd_host.Width;
-            return _hwnd_host;
-         }
-      }
-
-      void _try_set_wpf_bus() {
-         if (_owner is null) {
-            return;
-         }
-         _document.content.try_match(on_wpf_element);
-
-         void on_wpf_element(UIElement element) =>
-            element.Dispatcher.InvokeAsync(() =>
-               ui.WindowStyle.SetUnsafeWindow(element, _owner));
-      }
-   }
-}
+namespace mbts.ui.shell {
+   using bts;
+
+   using Spark.Core;
+   using Spark.Core.Async;
+   using Spark.Core.Disposables;
+
+   using System;
+   using System.Reactive.Linq;
+   using System.Runtime.InteropServices;
+   using System.Windows;
+   using System.Windows.Controls;
+   using System.Windows.Input;
+   using System.Windows.Media;
+   using System.Windows.Threading;
+
+   public class ShellWindowVM : DisposableViewModelBase {
+      readonly IWindowManager _window_manager;
+      readonly Property<bool> _edit_mode;
+      readonly Property<string> _title;
+      readonly Property<string> _title_override;
+      readonly Property<bool> _has_custom_title;
+
+      IShellWindow _model;
+      UIElement _ui;
+      ShellWindowUI _document;
+      HwndContentHost _hwnd_host;
+      FloatingWindow _owner;
+      ImageSource _icon;
+      ICommand _close_command;
+      ICommand _rename_command;
+      ICommand _unrename_command;
+
+      public ShellWindowVM(IShellWindow model, IWindowManager wm, IShellThreadFactory threads, string host = null) {
+         Log.assert(model != null);
+         _model = model;
+         _model.IconChanged += on_icon_changed;
+         _window_manager = wm;
+         host_id = (model as ITabAffineWindow)?.tab_group ?? host ?? string.Empty;
+
+         // init Dispatcher
+         var stf = (ShellThreadFactory)threads;
+         if (IsShared) {
+            Dispatcher = stf.get_dispatcher((ISharedWindow)model);
+         } else if (IsSlow) {
+            Dispatcher = stf.create_dispatcher((ISlowWindow)model);
+         } else {
+            Dispatcher = Dispatcher.CurrentDispatcher;
+         }
+
+         _edit_mode = backing(nameof(EditMode), false);
+
+         _title_override = backing<string>(nameof(TitleOverride));
+         _title = backing(
+            nameof(Title),
+            _model.observe_title()
+            .CombineLatest(
+               _title_override.observe_current(true),
+               (title, title_override) => (title, title_override))
+            .Select(t => t.title_override ?? t.title)
+            .OnDispatcher());
+         _has_custom_title = backing(
+            nameof(HasCustomTitle),
+            _title_override.observe_current(true).Select(t => !t.is_null_or_white_space()).OnDispatcher());
+
+         model.on_shell_window_vm(this);
+
+         _dispose = Disposable.Create(() => {
+            _model.IconChanged -= on_icon_changed;
+            if (IsShared) {
+               // Shared threads continue running
+               Dispatcher.Invoke(() => Disposable.dispose(ref _model));
+            } else if (IsSlow) {
+               // Terminating the background thread will dispose the model on that thread
+               Log.Ui.info("Beginning asyncronous dispose of slow window {0} .".fmt(_model.GetType().ToString()));
+               Dispatcher.BeginInvokeShutdown(DispatcherPriority.Background);
+            } else {
+               Disposable.dispose(ref _model);
+            }
+            _ui = null;
+            Disposable.dispose(ref _hwnd_host);
+         });
+
+         void on_icon_changed() {
+            _icon = null;
+            RaisePropertyChanged(nameof(Icon));
+         }
+      }
+
+      public AsyncEvent<(object sender, ShellWindowVM e)> Closing { get; } = new();
+
+      public string host_id { get; set; }
+      public string OwnerName => _model.Owner?.name ?? string.Empty;
+      public IShellWindow Model => _model;
+      public Dispatcher Dispatcher { get; }
+
+      // title
+      public string Title => _title.Value;
+      public string TitleOverride {
+         get => _title_override.Value;
+         set {
+            _title_override.Value = value;
+            _edit_mode.set(false);
+         }
+      }
+      public bool HasCustomTitle => _has_custom_title.Value;
+
+      public ImageSource Icon =>
+         _icon ??= _model.Icon ?? _model.Owner?.icon;
+
+      public bool EditMode => _edit_mode.Value;
+      public double InitialWidth { get; private set; }
+      public double InitialHeight { get; private set; }
+
+      public UIElement AppContent =>
+         _ui ??= _create_contents();
+
+      public bool IsSlow => _model is ISlowWindow;
+      public bool IsShared => _model is ISharedWindow;
+      public bool HasTabAffinity => _model is ITabAffineWindow;
+
+      public ICommand rename_command =>
+         _rename_command ??= new DelegateCommand(_begin_rename);
+
+      public ICommand unrename_command =>
+         _unrename_command ??= new DelegateCommand(() => TitleOverride = string.Empty);
+
+      public ICommand CloseCommand =>
+         _close_command ??= new DelegateCommand(_begin_close);
+
+      internal FloatingWindow owner {
+         get => _owner;
+         set {
+            if (Set(ref _owner, value)) {
+               _try_set_wpf_bus();
+            }
+         }
+      }
+
+      public IObservable<string> observe_title() =>
+         _title.observe_current(true);
+
+      void _begin_rename() {
+         if (string.IsNullOrEmpty(TitleOverride)) {
+            _title_override.Value = Title;
+         }
+         _edit_mode.set(true);
+      }
+
+      async void _begin_close(object sender) {
+         if (_window_manager is null) {
+            return;
+         }
+         if (!_window_manager.contains(_model)) {
+            return;
+         }
+         await Closing?.InvokeAsync((sender, this));
+         // Removing the window will cause this object to be Disposed
+         // Which will terminate the background thread
+         // Which will dispose the model (on that thread)
+         await _window_manager.remove(_model);
+      }
+
+      UIElement _create_contents() {
+         string error_text = null;
+
+         Dispatcher.Invoke(() => {
+            // Executes possibly on background thread
+            try {
+               _document = _model.GetContent();
+            } catch (Exception ex) {
+               error_text = ex.Message;
+               Log.Ui.error($"Error getting window contents: {ex}");
+            }
+         }, DispatcherPriority.Send);
+
+         if (_document == null && string.IsNullOrEmpty(error_text)) {
+            error_text = $"No content for window {_model.GetType()}";
+         }
+
+         // If there's an error draw a big red X
+         if (!error_text.is_null_or_empty()) {
+            _document = new ShellWindowUI(new TextBlock() {
+               HorizontalAlignment = HorizontalAlignment.Center,
+               VerticalAlignment = VerticalAlignment.Center,
+               Foreground = Brushes.Red,
+               Text = error_text
+            });
+            _model = new ErrorShellWindow {
+               Content = _document,
+               Icon = _model.Icon,
+               Title = $"Error: {_model.GetType()}"
+            };
+            RaisePropertyChanged("Title");
+         }
+
+         Log.assert(_document != null);
+         return _document.content.Match(on_wpf_element, on_handle_ref);
+
+         UIElement on_wpf_element(UIElement element) {
+            if (IsSlow || IsShared) {
+               // Create a WPFPanel (a Winforms window), then attach it with the HwndContentHost.
+               // This gives the document content a separate thread affinity from the shell window,
+               // and lets the win32 layer handle the thread boundary.
+               var retval = Dispatcher.Invoke(() => {
+                  var interop_window = new WPFPanel(element);
+                  element.SetCurrentValue(ui.WindowStyle.WinFormsHostProperty, interop_window);
+                  return new HandleRef(interop_window, interop_window.Handle);
+               }, DispatcherPriority.Send);
+               _hwnd_host = new HwndContentHost(retval);
+               InitialHeight = _hwnd_host.InitialHeight;
+               InitialWidth = _hwnd_host.InitialWidth;
+               return _hwnd_host;
+            }
+            if (element is FrameworkElement fe) {
+               InitialHeight = fe.Height;
+               InitialWidth = fe.Width;
+               fe.Height = fe.Width = double.NaN;
+            }
+            return element;
+         }
+
+         UIElement on_handle_ref(HandleRef handle_ref) {
+            _hwnd_host = new HwndContentHost(handle_ref);
+            InitialHeight = _hwnd_host.Height;
+            InitialWidth = _hwnd_host.Width;
+            return _hwnd_host;
+         }
+      }
+
+      void _try_set_wpf_bus() {
+         if (_owner is null) {
+            return;
+         }
+         _document.content.try_match(on_wpf_element);
+
+         void on_wpf_element(UIElement element) =>
+            element.Dispatcher.InvokeAsync(() =>
+               ui.WindowStyle.SetUnsafeWindow(element, _owner));
+      }
+   }
+}
@@ -1,5 +1,6 @@
 namespace bts.api {
   using System;
+   using System.Collections.Generic;
   using System.Linq;
   using System.Reflection.Metadata.Ecma335;

@@ -36,6 +37,15 @@ namespace bts.api {
         };
      }

+      public static IEnumerable<DateTime> normal_expirations(this SecurityImmutable security) =>
+         security.ContractCase switch {
+            Security.ContractOneofCase.Future => new[] { security.Future.expiration_date() },
+            Security.ContractOneofCase.Option => new[] { security.Option.expiration_date() },
+            Security.ContractOneofCase.Spread => security.Spread.Legs.SelectMany(l => l.Security.normal_expirations()),
+            Security.ContractOneofCase.Synthetic => security.Synthetic.Legs.SelectMany(l => l.Security.normal_expirations()),
+            _ => Enumerable.Empty<DateTime>()
+         };
+
      public static ProductImmutable product(this SecurityImmutable security) =>
         new(new() {
            Exchange = security.Exchange,
@@ -59,13 +69,7 @@ namespace bts.api {
         new((int)future.ExpirationYear, (int)future.ExpirationMonth, (int)future.ExpirationDay);

      internal static DateTime earliest_expiration(this SecurityImmutable security) =>
-         security.ContractCase switch {
-            Security.ContractOneofCase.Synthetic => security.Synthetic.Legs.First().Security.earliest_expiration(),
-            Security.ContractOneofCase.Spread => security.Spread.Legs.First().Security.earliest_expiration(),
-            Security.ContractOneofCase.Future => security.Future.expiration_date(),
-            Security.ContractOneofCase.Option => security.Option.expiration_date(),
-            _ => throw new NotImplementedException(),
-         };
+         security.normal_expirations().Min();

      public static double strike(this SecurityImmutable security) =>
         security.HasOption ? security.Option.Strike : double.NaN;
@@ -105,6 +109,7 @@ namespace bts.api {
            Security.ExchangeDataOneofCase.Deribit => security.Deribit.SecurityDesc,
            Security.ExchangeDataOneofCase.Ftx => security.Ftx.SecurityDesc,
            Security.ExchangeDataOneofCase.Binance => security.Binance.SecurityDesc,
+            Security.ExchangeDataOneofCase.BinanceCoinFutures => security.BinanceCoinFutures.SecurityDesc,
            Security.ExchangeDataOneofCase.BinanceUs => security.BinanceUs.SecurityDesc,
            Security.ExchangeDataOneofCase.BinanceUsdsFutures => security.BinanceUsdsFutures.SecurityDesc,
            Security.ExchangeDataOneofCase.Coinbase => security.Coinbase.SecurityDesc,
@@ -151,12 +156,13 @@ namespace bts.api {
            domain.Venue.Deribit => Venue.Deribit,
            domain.Venue.Ftx => Venue.Ftx,
            domain.Venue.Binance => Venue.Binance,
+            domain.Venue.BinanceCoinFutures => Venue.BinanceCoinFutures,
            domain.Venue.BinanceUs => Venue.BinanceUs,
            domain.Venue.BinanceUsdsFutures => Venue.BinanceUsdsFutures,
            domain.Venue.Glbx => Venue.Glbx,
            domain.Venue.Coinbase => Venue.Coinbase,
            domain.Venue.Okx => Venue.Okx,
-                        domain.Venue.Cboe => Venue.Cboe,
+            domain.Venue.Cboe => Venue.Cboe,
            domain.Venue.Taifex => Venue.Taifex,


@@ -171,6 +177,7 @@ namespace bts.api {
            Venue.Deribit => domain.Venue.Deribit,
            Venue.Ftx => domain.Venue.Ftx,
            Venue.Binance => domain.Venue.Binance,
+            Venue.BinanceCoinFutures => domain.Venue.BinanceCoinFutures,
            Venue.BinanceUs => domain.Venue.BinanceUs,
            Venue.BinanceUsdsFutures => domain.Venue.BinanceUsdsFutures,
            Venue.Coinbase => domain.Venue.Coinbase,
@@ -162,7 +162,7 @@
         return leg.signed_quantity() switch {
            1.0 => sec_str,
            -1.0 => $"-{sec_str}",
-            _ => ""
+            _ => $"{leg.signed_quantity()}{sec_str}",
         };
      }

@@ -172,7 +172,7 @@
         return leg.signed_quantity() switch {
            1.0 => $"+{sec_str}",
            -1.0 => $"-{sec_str}",
-            _ => $"{leg.signed_quantity():+0;-0}{sec_str}",
+            _ => $"{leg.signed_quantity():+0.######;-0.######}{sec_str}",
         };
      }
   }
@@ -0,0 +1,61 @@
+{
+  "user": "bts",
+  "pubsub": {
+    "milliseconds_between_sub_info_updates": 1000,
+    "milliseconds_between_time_slice_updates": 5000,
+    "milliseconds_between_no_buffer_space_retries": 100,
+    "retries_on_no_buffer_space": 1,
+    "do_it_all_seconds_between_logs": 0,
+    "heartbeat_interval": 15000,
+    "missing_heartbeats_before_gc": 50,
+    "use_managed_sockets": true,
+    "state_update_period": 5000,
+    "statistics_update_period": 5000,
+    "disposed_connection_check_period": 1000,
+    "message_processing_warning_threshold": 1000,
+    "server_port": 12345,
+    "publisher_routes": [
+      {
+        "name": "default",
+        "server_address": "bts-cloud://localhost:12345",
+        "tag": ".*"
+      }
+    ],
+    "subscriber_routes": [
+      {
+        "name": "default",
+        "server_address": "bts-cloud://localhost:12345",
+        "tag": ".*"
+      }
+    ],
+    "max_packet_size": 500000000,
+    "blue_comm_must_die": true,
+    "crushable_tags": [],
+    "tag_type_to_assembly_map": [
+      {
+        "handler_class": "BCG.PubSubServerTagHandler",
+        "assembly": "DataStreamCore.dll"
+      }
+    ],
+    "server_queue": "PubSubQueue",
+    "port_range": {
+      "first": 30400,
+      "last": 31700
+    },
+    "bind_to_random_port": false,
+    "debug_tags": "^NOTAGS$",
+    "latency_report_threshold_ms": 1500,
+    "use_substations": false,
+    "use_dynamic_clients": false,
+    "is_substation": false,
+    "internally_publish_sub_info": true,
+    "use_managed_clients_in_native_code": false,
+    "compressed_tags": "^NOTAGS$",
+    "tags_to_report_compression_ratio": "^NOTAGS$",
+    "disable_msmq": false,
+    "use_prepublished_server_state": false,
+    "ip_translation": null,
+    "gc_activation_threshold": 0,
+    "wait_period_after_gc": 1000
+  }
+}
@@ -1,4 +1,7 @@
 namespace spark_api_server {
+   using BCG;
+
+   using Bts.PubSub;
   using Bts.Websocket;

   using bts;
@@ -29,8 +32,11 @@

      static async Task<int> _run(RunOptions opts) {
         Console.WriteLine("Starting spark_api_server... ");
+         opts.MergeWithConfigFile();
+         opts.MergeWith(RunOptions.Default());
         Log.notnull(opts.user);
         Log.notnull(opts.password);
+         PubSubConfig.SetGlobalConfig(Log.notnull(opts.PubSub));
         var service_ctx = ServiceContext.create(opts.sparkd);
         await service_ctx.login(opts.user, opts.password);

@@ -67,6 +73,14 @@
            }

         });
+         var vsdTask = Task.Run(() => {
+            Log.Gen.info("Starting ViewStreamDatabase work loop");
+            var cancellationToken = cancellationTokenSource.Token;
+            while (!cancellationToken.IsCancellationRequested) {
+               ViewStreamDatabase.instance.doWork();
+               Thread.Sleep(100);
+            }
+         }, cancellationTokenSource.Token);
         // var task = Task.Run(() => {
         //    while (!cancellationTokenSource.IsCancellationRequested) {
         //       // Do something that can be interrupted
@@ -76,8 +90,11 @@
         // Console.WriteLine("Press any key to stop the server...");
         // Console.ReadKey();
         // cancellationTokenSource.Cancel();
-         await Task.WhenAny(listen);
+         await Task.WhenAny(listen, vsdTask);
         //UPTILL HERE
+         // These tasks aren't designed to finish naturally. If the code reaches here, it means one
+         // of them threw an exception. Cancel the other task to exit (somewhat) gracefully.
+         cancellationTokenSource.Cancel();
         return (int)errors;
      }
   }
@@ -1,6 +1,13 @@
 namespace Spark.Api.Server {
+   using Bts.PubSub;
+
   using CommandLine;

+   using System;
+   using System.IO;
+   using System.Text.Json;
+   using System.Text.Json.Serialization;
+
   [Verb("run")]
   public class RunOptions {
      [Option("user", Default = CommonOptionMetadata.DEFAULT_USER)]
@@ -14,5 +21,44 @@ namespace Spark.Api.Server {
         MetaValue = CommonOptionMetadata.SPARKD_META_VALUE,
         HelpText = CommonOptionMetadata.SPARKD_HELP_TEXT)]
      public string? sparkd { get; set; }
+
+      [JsonPropertyName("pubsub")]
+      public PubSubConfig? PubSub { get; set; }
+
+      [JsonIgnore]
+      [Option(
+         "config",
+         MetaValue = "PATH",
+         HelpText = "Path to json file representing these options (RunOptions)")]
+      public string? ConfigFile { get; set; }
+
+      public static RunOptions Default() =>
+         new() {
+            PubSub = PubSubConfig.Default(),
+         };
+
+      public void MergeWith(RunOptions other) {
+         sparkd ??= other.sparkd;
+         PubSub ??= other.PubSub;
+         ConfigFile ??= other.ConfigFile;
+      }
+
+      /// <summary>
+      /// Merges options with those defined in `ConfigFile` (does not overwrite
+      /// non-null existing values).
+      /// </summary>
+      /// <exception cref="InvalidOperationException">
+      /// JsonSerializer.Deserialize returns null
+      /// </exception>
+      public void MergeWithConfigFile() {
+         if (ConfigFile is null) {
+            return;
+         }
+         var cfgString = File.ReadAllText(ConfigFile);
+         var parsedCfg = JsonSerializer.Deserialize<RunOptions>(cfgString) ??
+            throw new InvalidOperationException($"Failed to parse json from {ConfigFile}");
+         MergeWith(parsedCfg);
+         ConfigFile = null;
+      }
   }
 }
@@ -0,0 +1,31 @@
+#nullable enable
+namespace bts;
+
+using System;
+
+public class ThrottledAction {
+   readonly TimeSpan interval;
+   readonly bool measureEndToNextStart;
+   DateTime lastInvokedTime = DateTime.MinValue;
+
+   public ThrottledAction(TimeSpan interval, bool measureEndToNextStart = true) {
+      this.interval = interval;
+      this.measureEndToNextStart = measureEndToNextStart;
+   }
+
+   public bool Execute(Action action) {
+      var now = DateTime.Now;
+      var elapsed = now - lastInvokedTime;
+      if (elapsed < interval) {
+         return false;
+      }
+      if (measureEndToNextStart) {
+         action();
+         lastInvokedTime = DateTime.Now;
+      } else {
+         lastInvokedTime = now;
+         action();
+      }
+      return true;
+   }
+}
@@ -50,6 +50,30 @@ namespace bts.utils {
      public static bool is_nan<T>(T value) where T : INumber<T> =>
         NormalNumericOps<T>.is_nan(value);

+      public static bool can_increment<T>(T value, T addend) where T : INumber<T>, IMinMaxValue<T> {
+         if (value > T.Zero != addend > T.Zero) {
+            return true;
+         }
+         if (value > T.Zero) {
+            return addend < T.MaxValue - value;
+         }
+         if (value < T.Zero) {
+            return addend > T.MinValue - value;
+         }
+         return true;
+      }
+
+      public static bool try_increment<T>(ref T value, T addend) where T : INumber<T>, IMinMaxValue<T> {
+         if (can_increment(value, addend)) {
+            value += addend;
+            return true;
+         }
+         return false;
+      }
+
+      public static bool try_increment(ref int value, long addend) =>
+         addend <= int.MaxValue && addend >= int.MinValue && try_increment(ref value, (int)addend);
+
      public static float gcd(IEnumerable<float> nums) =>
         gcd<float, NumericCastConverter>(nums);

@@ -1,14 +1,21 @@
 namespace bts.websocket_core {
+   using BCG;
+
   using bts.api;
   using bts.api.orderman;
   using bts.md;

+   using Bts.DataStream;
+   using Bts.DataStreamCore;
+   using Bts.DataStreamExpressions;
+
   using mbts.spark;

   using Newtonsoft.Json.Linq;

   using Spark.Api.MarketData;
   using Spark.Api.Orders;
+   using Spark.Core.Collections;

   using System;
   using System.Diagnostics.CodeAnalysis;
@@ -22,6 +29,7 @@ namespace bts.websocket_core {
         PRICE_STATS,
         PRICE_TICKERS,
         FILLS,
+         DATA_STREAM,
         UNKNOWN = 0
      }
      public bool Valid { get; set; } = true;
@@ -91,6 +99,47 @@ namespace bts.websocket_core {
               hdlr.SendMessage(WebsocketJsonParsers.to_json<OrderRecordTupleWithRoot, uint>(upd)));
         };
      }
+
+      private static Action<WebsocketSubscriptionHandler> GetDataStreamSubscriber(ViewStream vs, string subId) {
+         return Subscribe;
+
+         void Subscribe(WebsocketSubscriptionHandler hdlr) {
+            // Declare dictionaries here to reduce allocations in the callback. Use a lock
+            // to be safe (callbacks are likely single-threaded, this is just insurance).
+            var syncRoot = new object();
+            var removedRows = new Dictionary<int, object>();
+            var changedRows = new Dictionary<int, object>();
+            var addedRows = new Dictionary<int, object>();
+
+            hdlr.dis = vs.AsObservable().Subscribe(OnData);
+
+            void OnData(DataPacket dp) {
+               lock (syncRoot) {
+                  removedRows.Clear();
+                  changedRows.Clear();
+                  addedRows.Clear();
+                  foreach (IBlueRow r in dp.deletedRows) {
+                     var id = r.Id();
+                     removedRows[id] = r;
+                  }
+                  foreach (IBlueRow r in dp.addedRows) {
+                     var id = r.Id();
+                     if (removedRows.Remove(id)) {
+                        changedRows.Add(id, r);
+                     } else {
+                        addedRows.Add(id, r);
+                     }
+                  }
+                  hdlr.SendMessage(new {
+                     added = addedRows.Values,
+                     changed = changedRows.Values,
+                     removed = removedRows.Values,
+                  });
+               }
+            }
+         }
+      }
+
      private static Action<WebsocketSubscriptionHandler> GetFillsSubscriber(FillsSubscriptionFilter cri) {
         return (WebsocketSubscriptionHandler hdlr) => {
            hdlr.dis = hdlr._OrdersClient.fills(cri).Subscribe(
@@ -149,6 +198,39 @@ namespace bts.websocket_core {
         return true;
      }

+      static bool TryGetDataStream(
+         WebsocketRequest req,
+         [NotNullWhen(true)] out ViewStream? stream,
+         [NotNullWhen(false)] out string error
+      ) {
+         stream = null;
+         error = "";
+         var query = (req.parameters?["query"] as JValue)?.Value<string>();
+         if (query is null) {
+            error = "'params.query' field not found (expected a string)";
+            return false;
+         }
+         var exp = DataStreamExpressionsInterpreter.Evaluate(query);
+         if (exp is null) {
+            error = "'params.query' expression has invalid syntax";
+            return false;
+         }
+         Exception? exception = null;
+         try {
+            stream = exp.getView(new Dictionary<string, StatementExpression>(0));
+         } catch (Exception ex) {
+            exception = ex;
+         }
+         if (stream is null) {
+            error = "Failed to create ViewStream from 'params.query' expression (check for typos in arguments/typenames)";
+            if (exception is not null) {
+               error += ". " + exception.Message;
+            }
+            return false;
+         }
+         return true;
+      }
+
      static bool TryGetFilter(WebsocketRequest req, [NotNullWhen(true)] out SecurityCriteria? cri, out string message) {
         cri = null;
         if (req.parameters?["filter"] is not JObject filter) {
@@ -183,10 +265,13 @@ namespace bts.websocket_core {
      }

      public static bool TryCreate(
-         WebsocketRequest req, MarketDataClient md, OrdersClient oc,
+         WebsocketRequest req,
+         MarketDataClient md,
+         OrdersClient oc,
         Action<Object> sm,
         [NotNullWhen(true)] out WebsocketSubscriptionHandler? hdlr,
-         out string status) {
+         out string status
+      ) {
         hdlr = null;

         if (
@@ -247,6 +332,14 @@ namespace bts.websocket_core {
                  sub = GetOrdersSubscriber(filter);
                  break;
               }
+            case SUBSCRIPTION_TYPE.DATA_STREAM: {
+                  // TODO: make this nullable
+                  if (!TryGetDataStream(req, out var filter, out status)) {
+                     return false;
+                  }
+                  sub = GetDataStreamSubscriber(filter, id.Value.Val);
+                  break;
+               }
            default: {
                  status = "Incorrect subscription type";
                  return false;
@@ -19,13 +19,15 @@
      <PrivateAssets>all</PrivateAssets>
    </PackageReference>
    <PackageReference Include="System.Net.WebSockets" Version="4.3.0" />
-    <PackageReference Include="Newtonsoft.Json" Version="13.0.2" />
+    <PackageReference Include="Newtonsoft.Json" Version="13.0.3" />
    <PackageReference Include="Microsoft.AspNetCore.Cors" Version="2.2.0" />
  </ItemGroup>

  <ItemGroup>
    <ProjectReference Include="..\spark_core\spark_core.csproj" />
    <ProjectReference Include="..\spark_api\spark_api.csproj" />
+    <ProjectReference Include="..\..\DataStreamSuite\DataStream\DataStream.csproj" />
+    <ProjectReference Include="..\..\DataStreamSuite\SharedStreams\SharedStreams.csproj" />
  </ItemGroup>

  <ItemGroup>
@@ -1,4 +1,4 @@
-cmake_minimum_required(VERSION 3.5)
+cmake_minimum_required(VERSION 3.10)
 project(data_files)

 include(${CMAKE_SOURCE_DIR}/cmake/safe_dep_glob.cmake)
@@ -1,6 +1,7 @@
-cmake_minimum_required(VERSION 3.5)
+cmake_minimum_required(VERSION 3.10)
 project(libspark)
 set(CMAKE_SHARED_LIBRARY_PREFIX "")
+include(GNUInstallDirs)
 include(${CMAKE_SOURCE_DIR}/cmake/safe_dep_glob.cmake)
 include(${CMAKE_SOURCE_DIR}/cmake/Protobuf.cmake)
 include(${CMAKE_CURRENT_SOURCE_DIR}/cmake/GenerateDefaultConfig.cmake)
@@ -71,9 +72,7 @@ if (DEFINED DOTNET_WINDOWS_BINARY_DIR)
    "Copy ${PROJECT_NAME} bins to Dotnet Windows build dir")
 endif()

-if(BUILD_SPARKD)
-  install(TARGETS ${PROJECT_NAME} DESTINATION "${SPARK_INSTALL_PREFIX}/bin")
-endif()
+install(TARGETS ${PROJECT_NAME} DESTINATION "${CMAKE_INSTALL_BINDIR}")

 if(WIN32)
  include(${CMAKE_SOURCE_DIR}/cmake/vs_filters.cmake)
@@ -0,0 +1,34 @@
+syntax = "proto3";
+
+import "lib/api/domain/SecurityCriteria.proto";
+import "lib/api/core/RegexFormatting.proto";
+
+package bts.api.md;
+
+option csharp_namespace = "bts.api.md";
+option cc_enable_arenas = true;
+
+message BinanceCoinFuturesMarketDataConfig {
+  uint32 num_depth_levels = 1;
+  uint32 depth_throttle_ms = 2;
+  bts.api.domain.SecurityCriteria security_criteria = 3;
+  // Market data isn't subscribed for securities listed here (even if they are
+  // included by 'security_criteria'). There are some securities that don't
+  // fit into our system just yet, such as 'BTCBIDR', which trades in prices
+  // larger than our cpp 'domain::Price' class supports.
+  repeated string excluded_securities = 4;
+
+  // The list of instruments at Binance is obtained via an 'exchangeInfo' http
+  // rest request. In the corresponding response, each instrument has a 'pair'
+  // field, which is analgous to a security desc in our system (a unique
+  // human-readable string representation of the security). For some
+  // instruments, the string representation used in the market depth request is
+  // slightly different.
+  //
+  // 'depth_symbol_mappings' is a list of regex find/replace patterns to
+  // apply (to the 'pair' field) when subscribing to depth.
+  repeated bts.api.core.RegexFormatting security_desc_formattings = 5;
+
+  string url_rest_base = 6;
+  string url_ws_base = 7;
+}
@@ -1,7 +1,6 @@
 syntax = "proto3";

 import "lib/api/domain/SecurityCriteria.proto";
-import "lib/api/core/RegexFormatting.proto";

 package bts.api.md;

@@ -9,6 +8,13 @@ option csharp_namespace = "bts.api.md";
 option cc_enable_arenas = true;

 message BinanceUsdsFuturesMarketDataConfig {
+  message Subscription {
+    string symbol = 1;
+    string pair = 2;
+    string contract_type = 3;
+    string base_asset = 4;
+    string quote_asset = 5;
+  }
  uint32 num_depth_levels = 1;
  uint32 depth_throttle_ms = 2;
  bts.api.domain.SecurityCriteria security_criteria = 3;
@@ -17,17 +23,7 @@ message BinanceUsdsFuturesMarketDataConfig {
  // fit into our system just yet, such as 'BTCBIDR', which trades in prices
  // larger than our cpp 'domain::Price' class supports.
  repeated string excluded_securities = 4;
-
-  // The list of instruments at Binance is obtained via an 'exchangeInfo' http
-  // rest request. In the corresponding response, each instrument has a 'pair'
-  // field, which is analgous to a security desc in our system (a unique
-  // human-readable string representation of the security). For some
-  // instruments, the string representation used in the market depth request is
-  // slightly different.
-  //
-  // 'depth_symbol_mappings' is a list of regex find/replace patterns to
-  // apply (to the 'pair' field) when subscribing to depth.
-  repeated bts.api.core.RegexFormatting security_desc_formattings = 5;
+  repeated Subscription subscriptions = 8;

  string url_rest_base = 6;
  string url_ws_base = 7;
@@ -2,6 +2,7 @@ syntax = "proto3";

 import "lib/api/md/FTXMarketDataConfig.proto";
 import "lib/api/md/BinanceMarketDataConfig.proto";
+import "lib/api/md/BinanceCoinFuturesMarketDataConfig.proto";
 import "lib/api/md/BinanceUsMarketDataConfig.proto";
 import "lib/api/md/CoinbaseMarketDataConfig.proto";
 import "lib/api/md/BinanceUsdsFuturesMarketDataConfig.proto";
@@ -21,6 +22,7 @@ package bts.api.md;
 message MarketDataConfig {
  repeated FTXMarketDataConfig ftx = 1;
  repeated BinanceMarketDataConfig binance = 2;
+  repeated BinanceCoinFuturesMarketDataConfig binance_coin_futures = 11;
  repeated BinanceUsdsFuturesMarketDataConfig binance_usds_futures = 3;
  repeated BinanceUsMarketDataConfig binance_us = 4;
  repeated DeribitMarketDataConfig deribit = 5;
@@ -9,7 +9,9 @@ option cc_enable_arenas = true;
 // Config to control the recording of the market data for an exchange.
 // This config is part of MarketDataConfig.proto
 message MarketDataRecordingConfig {
-   uint32 num_of_seconds_to_run = 10;
+   uint32 num_of_seconds_to_run_file_writer=12;
+   uint32 num_of_seconds_to_run_db_writer=13;
   bts.api.Venue venue = 20;
   uint32 n_book_levels = 30;
+   uint32 batch_size = 40;
 }
@@ -66,6 +66,7 @@ bool AdminServiceUtils::has_exchange(
   case api::Venue::VENUE_DERIBIT: // TODO
   case api::Venue::VENUE_FTX:
   case api::Venue::VENUE_BINANCE:
+   case api::Venue::VENUE_BINANCE_COIN_FUTURES:
   case api::Venue::VENUE_BINANCE_US:
   case api::Venue::VENUE_COINBASE:
   case api::Venue::VENUE_BINANCE_USDS_FUTURES:
@@ -185,6 +186,7 @@ AdminServiceUtils::resolve_connection(
   case api::Venue::VENUE_FTX:
   case api::Venue::VENUE_OKX:
   case api::Venue::VENUE_BINANCE:
+   case api::Venue::VENUE_BINANCE_COIN_FUTURES:
   case api::Venue::VENUE_BINANCE_US:
   case api::Venue::VENUE_COINBASE:
   case api::Venue::VENUE_TAIFEX:
@@ -454,6 +456,7 @@ std::string AdminServiceUtils::find_account_exchange_account(
   case api::Venue::VENUE_DERIBIT: // TODO
   case api::Venue::VENUE_FTX:     // TODO
   case api::Venue::VENUE_BINANCE: // TODO
+   case api::Venue::VENUE_BINANCE_COIN_FUTURES:
   case api::Venue::VENUE_BINANCE_US:
   case api::Venue::VENUE_COINBASE:
   case api::Venue::VENUE_BINANCE_USDS_FUTURES:
@@ -580,6 +583,21 @@ bts::api::orderman::RoutingRecord AdminServiceUtils::get_routing_record(
      out_result.set_is_success(true);
      return rv;
   }
+   case api::VENUE_OKX: {
+      rv.set_session("okx_1");
+      rv.set_account(account_ptr->name());
+      rv.set_operator_id(user_ptr->name());
+      // AdminServiceUtils::ExchangeSubAccountResolver resolver;
+      // auto sub_acc_ptr = resolver.find_entry(&sec, account_ptr->name(), true);
+      // if (!sub_acc_ptr) {
+      //    out_result.set_is_success(false);
+      //    out_result.set_error_text("Could not resolve sub account");
+      //    return rv;
+      // }
+      // rv.set_account_code(sub_acc_ptr->exchange_subaccount());
+      out_result.set_is_success(true);
+      return rv;
+   }
   case api::Venue::VENUE_DERIBIT: {
      rv.set_session("deribit_1");
      rv.set_account(account_ptr->name());
@@ -749,12 +767,18 @@ bts::api::orderman::RoutingRecord AdminServiceUtils::get_routing_record(
      out_result.set_is_success(true);
      return rv;
   }
+   case api::VENUE_BINANCE_USDS_FUTURES: {
+      rv.set_session("binance_usds_futures_1");
+      rv.set_account(account_ptr->name());
+      rv.set_account(user_ptr->name());
+      out_result.set_is_success(true);
+      return rv;
+   }
   case api::VENUE_FTX:     // TODO
   case api::VENUE_BINANCE: // TODO
+   case api::VENUE_BINANCE_COIN_FUTURES:
   case api::VENUE_BINANCE_US:
   case api::Venue::VENUE_COINBASE:
-   case api::VENUE_BINANCE_USDS_FUTURES:
-   case api::VENUE_OKX:
   case api::Venue::VENUE_UNSET:
   case api::Venue::VENUE_BTS:
   case api::Venue::VENUE_CBOECFE:
@@ -1145,6 +1169,7 @@ std::string AdminServiceUtils::find_user_exchange_id(
   case api::Venue::VENUE_DERIBIT: // TODO
   case api::Venue::VENUE_FTX:     // TODO
   case api::Venue::VENUE_BINANCE: // TODO
+   case api::Venue::VENUE_BINANCE_COIN_FUTURES:
   case api::Venue::VENUE_BINANCE_US:
   case api::Venue::VENUE_COINBASE:
   case api::Venue::VENUE_BINANCE_USDS_FUTURES:
@@ -1180,6 +1205,7 @@ std::string AdminServiceUtils::find_user_company_id(
   case api::Venue::VENUE_DERIBIT:
   case api::Venue::VENUE_FTX:     // TODO
   case api::Venue::VENUE_BINANCE: // TODO
+   case api::Venue::VENUE_BINANCE_COIN_FUTURES:
   case api::Venue::VENUE_BINANCE_US:
   case api::Venue::VENUE_COINBASE:
   case api::Venue::VENUE_BINANCE_USDS_FUTURES:
@@ -41,6 +41,18 @@ class AdminServiceUtils {
         }
         return nullptr;
      }
+
+      bts::config::toggles::ExchangeSubAccountUnit const *find_entry(
+          const std::string &account, bool is_internal_account) {
+         for (const auto &esa : exch_sub_accounts.accounts()) {
+            const auto &a = is_internal_account ? esa.bts_account_name()
+                                                : esa.exchange_subaccount();
+            if (a == account) {
+               return &esa;
+            }
+         }
+         return nullptr;
+      }
   };
   ////////////////////////////////////////////
   // connection accessors ////////////////////
@@ -74,11 +74,6 @@ function(get_cpp_sources dest)
    expression/ParserExceptions.cpp
    expression/Value.cpp
    expression/tests/expression_parser_tests.cpp
-    io/http/CurlHandle.cpp
-    io/http/HTTPClient.cpp
-    io/http/HTTPHeaders.cpp
-    io/http/HTTPRequest.cpp
-    io/http/HTTPResponse.cpp
    logging/LoggerHub.cpp
    logging/Sink.cpp
    logging/logging.cpp
@@ -4,6 +4,7 @@ SecurityMetadataCollection {
      scope {
        clauses {
          symbols: "BTC"
+          symbols: "ETH"
          security_types: SECURITY_TYPE_FUTURE
          security_types: SECURITY_TYPE_SPREAD
          venues: VENUE_BINANCE
@@ -13,11 +14,11 @@ SecurityMetadataCollection {
      value {
        quantity_display {
          tick_factors {
-            tick_base: 1
-            tick_unit: 10
+            tick_base: 1000
+            tick_unit: 1
          }
          display_multiplier: NaN
-          show_decimal_part: false
+          show_decimal_part: true
          pad_with_leading_zeros: true
          show_zero_integer: true
          nan_text: ""
@@ -4028,6 +4028,10 @@ JetFIX {
         name: "deribit_1"
         venue: VENUE_DERIBIT
      }
+      connection {
+         name: "okx_1"
+         venue: VENUE_OKX
+      }
      # CME EXECUTION NEEDS TO BE UPDATED, probably below
      # connection {
      #    name: "onixs_sim"
@@ -19,6 +19,11 @@ SymbolLists {
      symbol: "BTC"
      symbol: "ETH"
    }
+    list {
+      key: "BINANCE_COIN_FUTURES"
+      symbol: "BTC"
+      symbol: "ETH"
+    }
    list {
      key: "TAIFEX"
      symbol: "QFFH"
@@ -140,6 +145,11 @@ Symbols {
      spot: @SymbolLists.list[BINANCE].symbol
      spread: @SymbolLists.list[BINANCE].symbol
    }
+    venues {
+      name: "VENUE_BINANCE_COIN_FUTURES"
+      future: @SymbolLists.list[BINANCE_COIN_FUTURES].symbol
+      spread: @SymbolLists.list[BINANCE_COIN_FUTURES].symbol
+    }
    venues {
      name: "VENUE_BINANCE_US"
      spot: @SymbolLists.list[BINANCE_US].symbol
@@ -13,6 +13,19 @@ MarketDataConfig {
      }
    }

+    binance_coin_futures {
+      num_depth_levels: 5
+      url_rest_base: "https://testnet.binancefuture.com"
+      url_ws_base: "wss://dstream.binance.com"
+      security_criteria {
+        clauses {
+          security_types: SECURITY_TYPE_FUTURE
+          symbols: "BTC"
+          symbols: "ETH"
+        }
+      }
+    }
+
    binance_us {
      num_depth_levels: 5
      url_rest_base: "https://api.binance.us"
@@ -13,6 +13,19 @@ MarketDataConfig {
      }
    }

+    binance_coin_futures {
+      num_depth_levels: 5
+      url_rest_base: "https://dapi.binance.com"
+      url_ws_base: "wss://dstream.binance.com"
+      security_criteria {
+        clauses {
+          security_types: SECURITY_TYPE_FUTURE
+          symbols: "BTC"
+          symbols: "ETH"
+        }
+      }
+    }
+
    binance_usds_futures {
      num_depth_levels: 5
      url_rest_base: "https://dapi.binance.com"
@@ -28,6 +28,9 @@ class TimeSpan {
   static constexpr TimeSpan from_micros(std::int64_t value);
   static constexpr TimeSpan from_millis(std::int64_t value);
   static constexpr TimeSpan from_seconds(std::int64_t value);
+   static constexpr TimeSpan from_minutes(std::int64_t value);
+   static constexpr TimeSpan from_hours(std::int64_t value);
+   static constexpr TimeSpan from_days(std::int64_t value);
   static TimeSpan from_posix_duration(boost::posix_time::time_duration t);
   static TimeSpan from_iso_string(const std::string &iso_string);

@@ -53,6 +56,8 @@ constexpr bool operator<(TimeSpan lhs, TimeSpan rhs) noexcept;
 constexpr bool operator<=(TimeSpan lhs, TimeSpan rhs) noexcept;
 constexpr bool operator>(TimeSpan lhs, TimeSpan rhs) noexcept;
 constexpr bool operator>=(TimeSpan lhs, TimeSpan rhs) noexcept;
+constexpr TimeSpan operator-(TimeSpan lhs, TimeSpan rhs) noexcept;
+constexpr TimeSpan operator+(TimeSpan lhs, TimeSpan rhs) noexcept;

 std::ostream &operator<<(std::ostream &out, const TimeSpan &ts) noexcept;

@@ -83,6 +88,21 @@ constexpr TimeSpan TimeSpan::from_seconds(std::int64_t value) {
   return TimeSpan{value * nanos_per_second};
 }

+constexpr TimeSpan TimeSpan::from_minutes(std::int64_t value) {
+   constexpr const auto nanos_per_minute = 60'000'000'000;
+   return TimeSpan{value * nanos_per_minute};
+}
+
+constexpr TimeSpan TimeSpan::from_hours(std::int64_t value) {
+   constexpr const auto nanos_per_hour = 3'600'000'000'000;
+   return TimeSpan{value * nanos_per_hour};
+}
+
+constexpr TimeSpan TimeSpan::from_days(std::int64_t value) {
+   constexpr const auto nanos_per_day = 86'400'000'000'000;
+   return TimeSpan{value * nanos_per_day};
+}
+
 constexpr std::int64_t TimeSpan::nanos() const { return ticks_; }

 constexpr std::int64_t TimeSpan::micros() const { return ticks_ / 1'000; }
@@ -130,6 +150,14 @@ constexpr bool operator>=(TimeSpan lhs, TimeSpan rhs) noexcept {
   return lhs.nanos() >= rhs.nanos();
 }

+constexpr TimeSpan operator-(TimeSpan lhs, TimeSpan rhs) noexcept {
+   return TimeSpan{lhs.nanos() - rhs.nanos()};
+}
+
+constexpr TimeSpan operator+(TimeSpan lhs, TimeSpan rhs) noexcept {
+   return TimeSpan{lhs.nanos() + rhs.nanos()};
+}
+
 } // namespace bts::core

 FMT_IMPL(bts::core::TimeSpan, t, "{}", t.to_iso_string());
@@ -5,12 +5,19 @@
 namespace bts::core {

 using std::uint32_t;
+using std::uint8_t;

-static bool is_valid_ymd(uint32_t year, uint32_t month, uint32_t day) {
+static bool is_valid_ymd(uint32_t year, uint8_t month, uint8_t day) {
   return year >= 1970 && month >= 1 && month <= 12 && day >= 1 && day <= 31;
 }

-TimeStamp::TimeStamp(uint32_t year, uint32_t month, uint32_t day) {
+TimeStamp::TimeStamp(
+    uint32_t year,
+    uint8_t month,
+    uint8_t day,
+    uint8_t hour,
+    uint8_t minute,
+    uint8_t second) {
   bts_assert_msgf(
       is_valid_ymd(year, month, day), "Invalid timestamp {}/{}/{} (y/m/d)",
       year, month, day);
@@ -18,6 +25,9 @@ TimeStamp::TimeStamp(uint32_t year, uint32_t month, uint32_t day) {
   date.tm_year = year - 1900;
   date.tm_mon = month - 1;
   date.tm_mday = day;
+   date.tm_hour = hour;
+   date.tm_min = minute;
+   date.tm_sec = second;
   auto time = std::chrono::system_clock::from_time_t(mktime(&date));
   auto duration = time.time_since_epoch();
   value_ =
@@ -25,7 +35,7 @@ TimeStamp::TimeStamp(uint32_t year, uint32_t month, uint32_t day) {
 }

 bool TimeStamp::from_ymd(
-    std::uint32_t year, std::uint32_t month, std::uint32_t day, TimeStamp *result) {
+    uint32_t year, uint8_t month, uint8_t day, TimeStamp *result) {
   if (!is_valid_ymd(year, month, day)) {
      return false;
   }
@@ -8,7 +8,6 @@
 #include <boost/date_time/posix_time/posix_time_duration.hpp>
 #include <boost/date_time/posix_time/ptime.hpp>
 #include <boost/date_time/time.hpp>
-#include <chrono>
 #include <cstdint>
 #include <fmt/format.h>
 #include <limits>
@@ -16,6 +15,9 @@

 namespace bts::core {

+#define VALID_YMD(year, month, day)                                            \
+   (year >= 1970 && month >= 1 && month <= 12 && day >= 1 && day <= 31)
+
 /**
 * @brief Represents time as nano-seconds since the unix epoch (Jan 1, 1970)
 */
@@ -23,8 +25,14 @@ struct TimeStamp {
 public:
   constexpr TimeStamp() = default;
   explicit constexpr TimeStamp(std::uint64_t value);
-   // constructs a TimeStamp from a UTC date
-   explicit TimeStamp(uint32_t year, uint32_t month, uint32_t day);
+   TimeStamp(std::uint32_t year, std::uint8_t month, std::uint8_t day);
+   TimeStamp(
+       std::uint32_t year,
+       std::uint8_t month,
+       std::uint8_t day,
+       std::uint8_t hour,
+       std::uint8_t minute,
+       std::uint8_t second);

   static constexpr TimeStamp min();
   static constexpr TimeStamp max();
@@ -36,10 +44,33 @@ struct TimeStamp {
   static constexpr TimeStamp from_seconds(std::int64_t value);
   static TimeStamp from_ptime(boost::posix_time::ptime t);
   static TimeStamp from_iso_string(const std::string &iso_string);
+
+   // Constructs a TimeStamp from a UTC date
+   static TimeStamp from_ymd(
+       std::uint32_t year, std::uint8_t month, std::uint8_t day);
+
   static bool from_ymd(
       std::uint32_t year,
-       std::uint32_t month,
-       std::uint32_t day,
+       std::uint8_t month,
+       std::uint8_t day,
+       TimeStamp *result);
+
+   static TimeStamp from_ymdhms(
+       std::uint32_t year,
+       std::uint8_t month,
+       std::uint8_t day,
+       std::uint8_t hour,
+       std::uint8_t minute,
+       std::uint8_t second);
+
+   // Constructs a TimeStamp from a UTC date
+   static bool from_ymdhms(
+       std::uint32_t year,
+       std::uint8_t month,
+       std::uint8_t day,
+       std::uint8_t hour,
+       std::uint8_t minute,
+       std::uint8_t second,
       TimeStamp *result);

   constexpr std::uint64_t nanos() const;
@@ -50,8 +81,11 @@ struct TimeStamp {
   constexpr double total_millis() const;
   constexpr double total_seconds() const;

-   constexpr std::int64_t to_int64();
-   inline void to_date(uint32_t &year, uint32_t &month, uint32_t &day);
+   constexpr std::int64_t to_int64() const;
+   std::int64_t to_date_int() const;
+   void to_date(uint32_t &year, uint32_t &month, uint32_t &day) const;
+   // Gets a TimeStamp representing this date at midnight.
+   TimeStamp date() const;
   constexpr boost::posix_time::ptime to_ptime() const;
   std::string to_iso_string() const;
   // current epoch ns
@@ -82,7 +116,9 @@ constexpr bool operator>=(TimeStamp lhs, TimeStamp rhs) noexcept;

 constexpr TimeSpan operator-(TimeStamp lhs, TimeStamp rhs) noexcept;
 constexpr TimeStamp operator-(TimeStamp lhs, TimeSpan rhs) noexcept;
+TimeStamp &operator-=(TimeStamp &lhs, TimeSpan rhs) noexcept;
 constexpr TimeStamp operator+(TimeStamp lhs, TimeSpan rhs) noexcept;
+TimeStamp &operator+=(TimeStamp &lhs, TimeSpan rhs) noexcept;

 std::ostream &operator<<(std::ostream &out, const TimeStamp &ts) noexcept;

@@ -92,6 +128,10 @@ std::ostream &operator<<(std::ostream &out, const TimeStamp &ts) noexcept;

 constexpr TimeStamp::TimeStamp(std::uint64_t value) : value_(value) {}

+inline TimeStamp::TimeStamp(
+    std::uint32_t year, std::uint8_t month, std::uint8_t day)
+    : TimeStamp(year, month, day, 0, 0, 0) {}
+
 constexpr TimeStamp TimeStamp::min() { return TimeStamp{0}; }

 constexpr TimeStamp TimeStamp::max() {
@@ -144,13 +184,21 @@ constexpr double TimeStamp::total_seconds() const {
   return static_cast<double>(value_) / 1000'000'000;
 }

-constexpr std::int64_t TimeStamp::to_int64() {
+constexpr std::int64_t TimeStamp::to_int64() const {
   bts_assert(
       value_ <=
       static_cast<std::int64_t>(std::numeric_limits<std::int64_t>::max()));
   return static_cast<std::int64_t>(value_);
 }
-void TimeStamp::to_date(uint32_t &year, uint32_t &month, uint32_t &day) {
+
+inline std::int64_t TimeStamp::to_date_int() const {
+   std::uint32_t y, m, d;
+   to_date(y, m, d);
+   return y << 6 | m << 4 | d;
+}
+
+inline void TimeStamp::to_date(
+    uint32_t &year, uint32_t &month, uint32_t &day) const {
   std::uint64_t epoch_secs = value_ / (1000 * 1000 * 1000);
   time_t ts_tt(epoch_secs);
   struct tm *ptm = gmtime(&ts_tt);
@@ -159,6 +207,12 @@ void TimeStamp::to_date(uint32_t &year, uint32_t &month, uint32_t &day) {
   day = ptm->tm_mday;
 }

+inline TimeStamp TimeStamp::date() const {
+   std::uint32_t y, m, d;
+   to_date(y, m, d);
+   return from_ymd(y, m, d);
+}
+
 constexpr boost::posix_time::ptime TimeStamp::to_ptime() const {
   constexpr const auto nanos_per_sec = 1'000'000'000;
   constexpr const auto nanos_per_micro = 1'000;
@@ -170,6 +224,36 @@ constexpr boost::posix_time::ptime TimeStamp::to_ptime() const {
          boost::posix_time::microseconds(nanos_rem / nanos_per_micro);
 }

+inline TimeStamp TimeStamp::from_ymd(
+    std::uint32_t year, std::uint8_t month, std::uint8_t day) {
+   return {year, month, day, 0, 0, 0};
+}
+
+inline TimeStamp TimeStamp::from_ymdhms(
+    std::uint32_t year,
+    std::uint8_t month,
+    std::uint8_t day,
+    std::uint8_t hour,
+    std::uint8_t minute,
+    std::uint8_t second) {
+   return {year, month, day, hour, minute, second};
+}
+
+inline bool TimeStamp::from_ymdhms(
+    std::uint32_t year,
+    std::uint8_t month,
+    std::uint8_t day,
+    std::uint8_t hour,
+    std::uint8_t minute,
+    std::uint8_t second,
+    TimeStamp *result) {
+   if (!VALID_YMD(year, month, day)) {
+      return false;
+   }
+   *result = TimeStamp{year, month, day, hour, minute, second};
+   return true;
+}
+
 constexpr bool operator==(TimeStamp lhs, TimeStamp rhs) noexcept {
   return lhs.nanos() == rhs.nanos();
 }
@@ -202,10 +286,20 @@ constexpr TimeStamp operator-(TimeStamp lhs, TimeSpan rhs) noexcept {
   return TimeStamp{lhs.nanos() - rhs.nanos()};
 }

+inline TimeStamp &operator-=(TimeStamp &lhs, TimeSpan rhs) noexcept {
+   lhs = lhs - rhs;
+   return lhs;
+}
+
 constexpr TimeStamp operator+(TimeStamp lhs, TimeSpan rhs) noexcept {
   return TimeStamp{lhs.nanos() + rhs.nanos()};
 }

+inline TimeStamp &operator+=(TimeStamp &lhs, TimeSpan rhs) noexcept {
+   lhs = lhs + rhs;
+   return lhs;
+}
+
 } // namespace bts::core

 FMT_IMPL(bts::core::TimeStamp, t, "{}", t.to_iso_string());
@@ -43,8 +43,6 @@ class BookT {

   void reset_merged_quantity();
   void reset_n_orders();
-   template <class Container>
-   void apply_changes(Side side, const Container &changes);

   /**
    * Removes price levels of the non-dominant side that cross the dominant
@@ -203,12 +201,6 @@ bool BookT<BS>::is_sorted() const {
   return Side::forboth([&](Side s) { return operator[](s).is_sorted(s); });
 }

-template <class BS>
-template <class Container>
-void BookT<BS>::apply_changes(Side side, const Container &changes) {
-   _books[static_cast<std::size_t>(side.value)].apply_changes(side, changes);
-}
-
 template <class BS>
 void from_json(const nlohmann::json &j, BookT<BS> &book) {
   io::JsonReader jr{&j};
@@ -33,6 +33,8 @@ api::domain::Venue DomainSecurityUtils::convert(bts::api::Venue venue) {
      return DOMAIN_VENUE(SIM);
   case bts::api::Venue::VENUE_BINANCE:
      return DOMAIN_VENUE(BINANCE);
+   case bts::api::Venue::VENUE_BINANCE_COIN_FUTURES:
+      return DOMAIN_VENUE(BINANCE_COIN_FUTURES);
   case bts::api::Venue::VENUE_BINANCE_US:
      return DOMAIN_VENUE(BINANCE_US);
   case bts::api::VENUE_BINANCE_USDS_FUTURES:
@@ -65,6 +67,8 @@ bts::api::Venue DomainSecurityUtils::convert(api::domain::Venue venue) {
      return api::Venue::VENUE_FTX;
   case DOMAIN_VENUE(BINANCE):
      return api::Venue::VENUE_BINANCE;
+   case DOMAIN_VENUE(BINANCE_COIN_FUTURES):
+      return api::Venue::VENUE_BINANCE_COIN_FUTURES;
   case DOMAIN_VENUE(BINANCE_US):
      return api::Venue::VENUE_BINANCE_US;
   case DOMAIN_VENUE(BINANCE_USDS_FUTURES):
@@ -166,8 +170,6 @@ std::string DomainSecurityUtils::get_exchange_id(const bts::api::Security &s) {
 bool DomainSecurityUtils::get_security_desc(
    const bts::api::Security &security, std::string &desc) {
   switch (security.ExchangeData_case()) {
-   case bts::api::Security::ExchangeDataCase::EXCHANGEDATA_NOT_SET:
-      bts_assert(false);
   case bts::api::Security::ExchangeDataCase::kCboe:
      desc = security.cboe().security_desc();
      break;
@@ -189,6 +191,9 @@ bool DomainSecurityUtils::get_security_desc(
   case bts::api::Security::ExchangeDataCase::kBinance:
      desc = security.binance().security_desc();
      break;
+   case bts::api::Security::ExchangeDataCase::kBinanceCoinFutures:
+      desc = security.binance_coin_futures().security_desc();
+      break;
   case bts::api::Security::ExchangeDataCase::kBinanceUs:
      desc = security.binance_us().security_desc();
      break;
@@ -205,6 +210,10 @@ bool DomainSecurityUtils::get_security_desc(
      desc = security.ftx().security_desc();
      break;
   default:
+      if (security.has_synthetic()) {
+         desc = security.synthetic().name();
+         break;
+      }
      return false;
   }
   return true;
@@ -3,6 +3,7 @@
 #include "Side.h"
 #include "lib/core/FMT_IMPL.h"
 #include "lib/io/JsonReader.h"
+#include "lib/utils/bts_array_size.h"
 #include "lib/utils/bts_assert.h"

 #include <cmath>
@@ -33,6 +34,22 @@

 namespace bts::domain {

+static const double powers_10[] = {
+    1E-20, 1E-19, 1E-18, 1E-17, 1E-16, 1E-15, 1E-14, 1E-13, 1E-12, 1E-11, 1E-10,
+    1E-9,  1E-8,  1E-7,  1E-6,  1E-5,  1E-4,  1E-3,  1E-2,  1E-1,  1E0,   1E1,
+    1E2,   1E3,   1E4,   1E5,   1E6,   1E7,   1E8,   1E9,   1E10,  1E11,  1E12,
+    1E13,  1E14,  1E15,  1E16,  1E17,  1E18,  1E19,  1E20};
+static const int max_abs_power_10 = BTS_ARRAYSIZE(powers_10) / 2;
+inline double pow10(int power) {
+   const int offset = power + max_abs_power_10;
+   const int powers_10_sz = BTS_ARRAYSIZE(powers_10);
+   bts_assert_d(offset >= 0 && offset < powers_10_sz);
+   if (offset >= 0 && offset < powers_10_sz) {
+      return powers_10[offset];
+   }
+   return std::numeric_limits<double>::quiet_NaN();
+}
+
 class Price {
 public:
   Price() = default;
@@ -120,6 +137,40 @@ class Price {
   constexpr Price &operator*=(Price factor) noexcept;
   constexpr Price &operator/=(Price factor) noexcept;

+   constexpr bool operator==(double factor) const noexcept;
+   constexpr bool operator!=(double factor) const noexcept;
+   constexpr bool operator<(double factor) const noexcept;
+   constexpr bool operator<=(double factor) const noexcept;
+   constexpr bool operator>(double factor) const noexcept;
+   constexpr bool operator>=(double factor) const noexcept;
+
+   constexpr Price operator+(double factor) const noexcept;
+   constexpr Price operator-(double factor) const noexcept;
+   constexpr Price operator*(double factor) const noexcept;
+   constexpr Price operator/(double factor) const noexcept;
+
+   constexpr Price &operator+=(double factor) noexcept;
+   constexpr Price &operator-=(double factor) noexcept;
+   constexpr Price &operator*=(double factor) noexcept;
+   constexpr Price &operator/=(double factor) noexcept;
+
+   constexpr bool operator==(std::int32_t factor) const noexcept;
+   constexpr bool operator!=(std::int32_t factor) const noexcept;
+   constexpr bool operator<(std::int32_t factor) const noexcept;
+   constexpr bool operator<=(std::int32_t factor) const noexcept;
+   constexpr bool operator>(std::int32_t factor) const noexcept;
+   constexpr bool operator>=(std::int32_t factor) const noexcept;
+
+   constexpr Price operator+(std::int32_t factor) const noexcept;
+   constexpr Price operator-(std::int32_t factor) const noexcept;
+   constexpr Price operator*(std::int32_t factor) const noexcept;
+   constexpr Price operator/(std::int32_t factor) const noexcept;
+
+   constexpr Price &operator+=(std::int32_t factor) noexcept;
+   constexpr Price &operator-=(std::int32_t factor) noexcept;
+   constexpr Price &operator*=(std::int32_t factor) noexcept;
+   constexpr Price &operator/=(std::int32_t factor) noexcept;
+
   constexpr bool operator==(std::int64_t factor) const noexcept;
   constexpr bool operator!=(std::int64_t factor) const noexcept;
   constexpr bool operator<(std::int64_t factor) const noexcept;
@@ -431,7 +482,8 @@ inline void to_json(nlohmann::json &j, const Price &price) {
      if (is_nan() || factor.is_nan()) {                                       \
         return nan();                                                         \
      }                                                                        \
-      return Price{_mantissa op factor._mantissa};                             \
+      return Price{_mantissa op static_cast<std::int64_t>(                     \
+          factor._mantissa * pow10(factor.exponent() - exponent()))};          \
   }

 // NOLINTNEXTLINE
@@ -440,7 +492,103 @@ inline void to_json(nlohmann::json &j, const Price &price) {
      if (factor.is_nan()) {                                                   \
         _mantissa = nan()._mantissa;                                          \
      } else if (!is_nan()) {                                                  \
-         _mantissa op factor._mantissa;                                        \
+         _mantissa op static_cast<std::int64_t>(                               \
+             factor._mantissa * pow10(factor.exponent() - exponent()));        \
+      }                                                                        \
+      return *this;                                                            \
+   }
+
+// NOLINTNEXTLINE
+#define PRICE_DOUBLE_COMPARISON(op, nan_part)                                  \
+   constexpr bool Price::operator op(double factor) const noexcept {           \
+      return nan_part _mantissa op factor;                                     \
+   }
+
+// NOLINTNEXTLINE
+#define PRICE_DOUBLE_BIN_OP(op)                                                \
+   constexpr Price Price::operator op(double factor) const noexcept {          \
+      if (is_nan() || std::isnan(factor)) {                                    \
+         return nan();                                                         \
+      }                                                                        \
+      return Price{static_cast<std::int64_t>(_mantissa op factor)};            \
+   }
+
+// NOLINTNEXTLINE
+#define PRICE_DOUBLE_EXP_BIN_OP(op)                                            \
+   constexpr Price Price::operator op(double factor) const noexcept {          \
+      if (is_nan() || std::isnan(factor)) {                                    \
+         return nan();                                                         \
+      }                                                                        \
+      return Price{_mantissa op static_cast<std::int64_t>(                     \
+          factor * pow10(-exponent()))};                                       \
+   }
+
+// NOLINTNEXTLINE
+#define PRICE_DOUBLE_BIN_REF_OP(op)                                            \
+   constexpr Price &Price::operator op(double factor) noexcept {               \
+      if (!is_nan()) {                                                         \
+         if (std::isnan(factor)) {                                             \
+            *this = nan();                                                     \
+         } else {                                                              \
+            _mantissa op factor;                                               \
+         }                                                                     \
+      }                                                                        \
+      return *this;                                                            \
+   }
+
+// NOLINTNEXTLINE
+#define PRICE_DOUBLE_EXP_BIN_REF_OP(op)                                        \
+   constexpr Price &Price::operator op(double factor) noexcept {               \
+      if (!is_nan()) {                                                         \
+         if (std::isnan(factor)) {                                             \
+            *this = nan();                                                     \
+         } else {                                                              \
+            _mantissa op factor *pow10(-exponent());                           \
+         }                                                                     \
+      }                                                                        \
+      return *this;                                                            \
+   }
+
+// NOLINTNEXTLINE
+#define PRICE_INT32_COMPARISON(op, nan_part)                                   \
+   constexpr bool Price::operator op(std::int32_t factor) const noexcept {     \
+      return nan_part _mantissa op static_cast<std::int64_t>(                  \
+          factor * pow10(-exponent()));                                        \
+   }
+
+// NOLINTNEXTLINE
+#define PRICE_INT32_BIN_OP(op)                                                 \
+   constexpr Price Price::operator op(std::int32_t factor) const noexcept {    \
+      if (is_nan()) {                                                          \
+         return *this;                                                         \
+      }                                                                        \
+      return Price{_mantissa op static_cast<std::int64_t>(                     \
+          factor * pow10(-exponent()))};                                       \
+   }
+
+#define PRICE_INT32_EXP_BIN_OP(op)                                             \
+   constexpr Price Price::operator op(std::int32_t factor) const noexcept {    \
+      if (is_nan()) {                                                          \
+         return *this;                                                         \
+      }                                                                        \
+      return Price{_mantissa op static_cast<std::int64_t>(                     \
+          factor * pow10(-exponent()))};                                       \
+   }
+
+// NOLINTNEXTLINE
+#define PRICE_INT32_BIN_REF_OP(op)                                             \
+   constexpr Price &Price::operator op(std::int32_t factor) noexcept {         \
+      if (!is_nan()) {                                                         \
+         _mantissa op factor;                                                  \
+      }                                                                        \
+      return *this;                                                            \
+   }
+
+// NOLINTNEXTLINE
+#define PRICE_INT32_EXP_BIN_REF_OP(op)                                         \
+   constexpr Price &Price::operator op(std::int32_t factor) noexcept {         \
+      if (!is_nan()) {                                                         \
+         _mantissa op static_cast<std::int64_t>(factor * pow10(-exponent()));  \
      }                                                                        \
      return *this;                                                            \
   }
@@ -460,6 +608,15 @@ inline void to_json(nlohmann::json &j, const Price &price) {
      return Price{_mantissa op factor};                                       \
   }

+#define PRICE_INT64_EXP_BIN_OP(op)                                             \
+   constexpr Price Price::operator op(std::int64_t factor) const noexcept {    \
+      if (is_nan()) {                                                          \
+         return *this;                                                         \
+      }                                                                        \
+      return Price{_mantissa op(                                               \
+          factor * static_cast<std::int64_t>(pow10(-exponent())))};            \
+   }
+
 // NOLINTNEXTLINE
 #define PRICE_INT64_BIN_REF_OP(op)                                             \
   constexpr Price &Price::operator op(std::int64_t factor) noexcept {         \
@@ -469,6 +626,15 @@ inline void to_json(nlohmann::json &j, const Price &price) {
      return *this;                                                            \
   }

+// NOLINTNEXTLINE
+#define PRICE_INT64_EXP_BIN_REF_OP(op)                                         \
+   constexpr Price &Price::operator op(std::int64_t factor) noexcept {         \
+      if (!is_nan()) {                                                         \
+         _mantissa op static_cast<std::int64_t>(factor * pow10(-exponent()));  \
+      }                                                                        \
+      return *this;                                                            \
+   }
+
 PRICE_PRICE_COMPARISON(!=, is_nan() ||)
 PRICE_PRICE_COMPARISON(==, !is_nan() &&)
 PRICE_PRICE_COMPARISON(<, !is_nan() &&)
@@ -486,6 +652,40 @@ PRICE_PRICE_BIN_REF_OP(-=)
 PRICE_PRICE_BIN_REF_OP(*=)
 PRICE_PRICE_BIN_REF_OP(/=)

+PRICE_DOUBLE_COMPARISON(!=, is_nan() ||)
+PRICE_DOUBLE_COMPARISON(==, !is_nan() &&)
+PRICE_DOUBLE_COMPARISON(<, !is_nan() &&)
+PRICE_DOUBLE_COMPARISON(<=, !is_nan() &&)
+PRICE_DOUBLE_COMPARISON(>, !is_nan() &&)
+PRICE_DOUBLE_COMPARISON(>=, !is_nan() &&)
+
+PRICE_DOUBLE_EXP_BIN_OP(+)
+PRICE_DOUBLE_EXP_BIN_OP(-)
+PRICE_DOUBLE_BIN_OP(*)
+PRICE_DOUBLE_BIN_OP(/)
+
+PRICE_DOUBLE_EXP_BIN_REF_OP(+=)
+PRICE_DOUBLE_EXP_BIN_REF_OP(-=)
+PRICE_DOUBLE_BIN_REF_OP(*=)
+PRICE_DOUBLE_BIN_REF_OP(/=)
+
+PRICE_INT32_COMPARISON(!=, is_nan() ||)
+PRICE_INT32_COMPARISON(==, !is_nan() &&)
+PRICE_INT32_COMPARISON(<, !is_nan() &&)
+PRICE_INT32_COMPARISON(<=, !is_nan() &&)
+PRICE_INT32_COMPARISON(>, !is_nan() &&)
+PRICE_INT32_COMPARISON(>=, !is_nan() &&)
+
+PRICE_INT32_EXP_BIN_OP(+)
+PRICE_INT32_EXP_BIN_OP(-)
+PRICE_INT32_BIN_OP(*)
+PRICE_INT32_BIN_OP(/)
+
+PRICE_INT32_EXP_BIN_REF_OP(+=)
+PRICE_INT32_EXP_BIN_REF_OP(-=)
+PRICE_INT32_BIN_REF_OP(*=)
+PRICE_INT32_BIN_REF_OP(/=)
+
 PRICE_INT64_COMPARISON(!=, is_nan() ||)
 PRICE_INT64_COMPARISON(==, !is_nan() &&)
 PRICE_INT64_COMPARISON(<, !is_nan() &&)
@@ -493,13 +693,13 @@ PRICE_INT64_COMPARISON(<=, !is_nan() &&)
 PRICE_INT64_COMPARISON(>, !is_nan() &&)
 PRICE_INT64_COMPARISON(>=, !is_nan() &&)

-PRICE_INT64_BIN_OP(+)
-PRICE_INT64_BIN_OP(-)
+PRICE_INT64_EXP_BIN_OP(+)
+PRICE_INT64_EXP_BIN_OP(-)
 PRICE_INT64_BIN_OP(*)
 PRICE_INT64_BIN_OP(/)

-PRICE_INT64_BIN_REF_OP(+=)
-PRICE_INT64_BIN_REF_OP(-=)
+PRICE_INT64_EXP_BIN_REF_OP(+=)
+PRICE_INT64_EXP_BIN_REF_OP(-=)
 PRICE_INT64_BIN_REF_OP(*=)
 PRICE_INT64_BIN_REF_OP(/=)

@@ -14,6 +14,7 @@ namespace securities {

 using bts::api::Security;
 using bts::api::Venue;
+using std::optional;

 api::SecurityType security_type(api::Security const &security) {
   return security_type(security.Contract_case());
@@ -37,7 +38,7 @@ api::SecurityType security_type(api::Security::ContractCase contract_case) {
   return api::SecurityType::SECURITYTYPE_UNSET;
 }

-boost::gregorian::date smallest_leg(
+std::optional<core::TimeStamp> smallest_leg(
    ::google::protobuf::RepeatedPtrField<::bts::api::ContractLeg> const &l) {
   bts_assert(l.size());
   auto smallest = expiration(l[0].security());
@@ -50,49 +51,59 @@ boost::gregorian::date smallest_leg(
   return smallest;
 }

-boost::gregorian::date expiration(api::Security const &security) {
+std::optional<core::TimeStamp> expiration(api::Security const &security) {
   switch (security.Contract_case()) {
-   case api::Security::kFuture:
-      return {
-          boost::gregorian::greg_year(security.future().expiration_year()),
-          boost::gregorian::greg_month(security.future().expiration_month()),
-          boost::gregorian::greg_day(security.future().expiration_day())};
-   case api::Security::kOption:
-      return {
-          boost::gregorian::greg_year(security.option().expiration_year()),
-          boost::gregorian::greg_month(security.option().expiration_month()),
-          boost::gregorian::greg_day(security.option().expiration_day())};
+   case api::Security::kFuture: {
+      const auto &fut = security.future();
+      core::TimeStamp exp;
+      if (core::TimeStamp::from_ymd(
+              fut.expiration_year(), fut.expiration_month(),
+              fut.expiration_day(), &exp)) {
+         return exp;
+      }
+      break;
+   }
+   case api::Security::kOption: {
+      const auto &opt = security.option();
+      core::TimeStamp exp;
+      if (core::TimeStamp::from_ymd(
+              opt.expiration_year(), opt.expiration_month(),
+              opt.expiration_day(), &exp)) {
+         return exp;
+      }
+      break;
+   }
   case api::Security::kSpread:
      return smallest_leg(security.spread().legs());
   case api::Security::kSynthetic:
      return smallest_leg(security.synthetic().legs());
   default:
-      return {};
+      break;
   }
+   return std::nullopt;
 }

 bool is_expired(
    api::Security const &security,
-    datetime::PTime const &now,
+    core::TimeStamp now,
    uint32_t sod_hh,
    uint32_t sod_mm) {
-   auto e = expiration(security);
-   if (e == boost::gregorian::date())
+   auto maybe_exp = expiration(security);
+   if (!maybe_exp) {
      return false;
-   else {
-      auto etime = datetime::PTime{
-          e, boost::posix_time::time_duration(sod_hh, sod_mm, 0)};
-      return etime < now;
   }
+   const auto exp = *maybe_exp;
+   return exp < now && (exp.date() + core::TimeSpan::from_hours(sod_hh) +
+                        core::TimeSpan::from_minutes(sod_mm)) < now;
 }

-bool is_expired(api::Security const &security, datetime::PTime const &now) {
+bool is_expired(api::Security const &security, core::TimeStamp now) {
   const auto &sod = config::get<config::SODTime>();
   return is_expired(security, now, sod.sod_hh(), sod.sod_mm());
 }

 bool is_expired(api::Security const &security) {
-   return is_expired(security, datetime::local_now());
+   return is_expired(security, core::TimeStamp::now());
 }

 void write_date_string(
@@ -237,13 +237,13 @@ round_to_quote_price(api::Security const &sec, Price in, api::Side side) {
          raw_round_to_quote_price(p, sec.contract_spec().low_tick(), side));
 }

-boost::gregorian::date expiration(api::Security const &sec);
+std::optional<core::TimeStamp> expiration(api::Security const &sec);
 bool is_expired(api::Security const &security);
 // for testing
-bool is_expired(api::Security const &security, datetime::PTime const &now);
+bool is_expired(api::Security const &security, core::TimeStamp now);
 bool is_expired(
    api::Security const &security,
-    datetime::PTime const &now,
+    core::TimeStamp now,
    uint32_t sod_hh,
    uint32_t sod_mm);

@@ -15,6 +15,7 @@ enum Venue {
     VENUE_DERIBIT = 400;
     VENUE_FTX = 410;
     VENUE_BINANCE = 420;
+     VENUE_BINANCE_COIN_FUTURES = 425;
     VENUE_BINANCE_USDS_FUTURES = 430;
     VENUE_BINANCE_US = 440;
     VENUE_COINBASE = 450;
@@ -106,6 +107,11 @@ message BinanceExchangeData {
  string security_desc = 2;
 }

+message BinanceCoinFuturesExchangeData {
+  int64 exchange_id = 1;
+  string security_desc = 2;
+}
+
 message BinanceUsdsFuturesExchangeData {
  int64 exchange_id = 1;
  string security_desc = 2;
@@ -164,12 +170,20 @@ message OptionContract {
 message SpotContract{
 }

+// his tells the relationshio of a leg with thr previous leg
+enum PriceCalc {
+		ADD = 0;
+		MULT = 1;
+        Div = 2;
+	}
+
 message ContractLeg {
    Security security = 10;
    double quantity = 20;
    Side side = 30;
    double price_multiplier = 40;
    double leg_delta = 50;
+    PriceCalc price_calc = 60;
 }

 message SpreadContract {    
@@ -228,6 +242,7 @@ message Security {
       DeribitExchangeData deribit = 240;
       FTXExchangeData ftx = 250;
       BinanceExchangeData binance = 260;
+       BinanceCoinFuturesExchangeData binance_coin_futures = 265;
       BinanceUsdsFuturesExchangeData binance_usds_futures = 270;
       BinanceUsExchangeData binance_us = 280;
       CoinbaseExchangeData coinbase = 290;
@@ -1,15 +1,22 @@
 #pragma once

-#include "lib/domain_model/Security.pb.h"
-#include "lib/core/FMT_IMPL.h"
 #include "lib/core/EnumMacros.h"
+#include "lib/core/FMT_IMPL.h"
+#include "lib/domain_model/Security.pb.h"
 #include "lib/io/JsonReader.h"
 #include "lib/logging/logging.h"
+#include "lib/utils/string_utils.h"

 #include <nlohmann/json.hpp>

 namespace bts::api {

+/**
+ * Parses Venue from a string. Unlike `bts::api::SecurityType_Parse`, this implementation is case
+ * insensitive, and makes the 'SECURITYTYPE_' prefix optional.
+ */
+bool security_type_parse_smart(const std::string &str, SecurityType *result);
+
 PB_ENUM_FROM_JSON(SecurityType)
 PB_ENUM_TO_JSON(SecurityType)

@@ -19,6 +26,22 @@ std::ostream &operator<<(std::ostream &out, const SecurityType &type) noexcept;
 // Definition
 //////////////////////////////////////////////////

+inline bool security_type_parse_smart(
+    const std::string &str, SecurityType *result) {
+   if (SecurityType_Parse(str, result)) {
+      return true;
+   }
+   std::string s = bts::system::string_utils::copy_to_upper(str);
+   if (SecurityType_Parse(s, result)) {
+      return true;
+   }
+   const char *prefix = "SECURITYTYPE_";
+   if (!s.starts_with(prefix)) {
+      s.insert(0, prefix);
+   }
+   return SecurityType_Parse(s, result);
+}
+
 inline std::ostream &operator<<(
    std::ostream &out, const SecurityType &type) noexcept {
   out << SecurityType_Name(type);
@@ -5,11 +5,18 @@
 #include "lib/domain_model/Security.pb.h"
 #include "lib/io/JsonReader.h"
 #include "lib/logging/logging.h"
+#include "lib/utils/string_utils.h"

 #include <nlohmann/json.hpp>

 namespace bts::api {

+/**
+ * Parses Venue from a string. Unlike `bts::api::Venue_Parse`, this implementation is case
+ * insensitive, and makes the 'VENUE_' prefix optional.
+ */
+bool venue_parse_smart(const std::string &str, Venue *result);
+
 PB_ENUM_FROM_JSON(Venue)
 PB_ENUM_TO_JSON(Venue)

@@ -19,6 +26,21 @@ std::ostream &operator<<(std::ostream &out, Venue venue) noexcept;
 // Definition
 //////////////////////////////////////////////////

+inline bool venue_parse_smart(const std::string &str, Venue *result) {
+   if (Venue_Parse(str, result)) {
+      return true;
+   }
+   std::string s = bts::system::string_utils::copy_to_upper(str);
+   if (Venue_Parse(s, result)) {
+      return true;
+   }
+   const char *prefix = "VENUE_";
+   if (!s.starts_with(prefix)) {
+      s.insert(0, prefix);
+   }
+   return Venue_Parse(s, result);
+}
+
 inline std::ostream &operator<<(std::ostream &out, Venue venue) noexcept {
   out << Venue_Name(venue);
   return out;
@@ -116,4 +116,4 @@ using UserLogin = utils::ValueWrapper<struct UserLoginTag, std::string>;
 } // namespace bts

 FMT_IMPL(bts::Price, p, "{}", p.value());
-FMT_IMPL(bts::Quantity, q, "{}", q.value());
+FMT_IMPL(bts::Quantity, q, "{}", q.full_value());
@@ -210,37 +210,22 @@ auto fut_text = "      id: 6\n"

 TEST_CASE("expiration_tests", "[security]") {
   SECTION("test_expiration_retrieval") {
-
-      using namespace boost::gregorian;
-
      auto spread = utils::pb::from_string<api::Security>(spread_txt);
-      REQUIRE(
-          securities::expiration(spread) ==
-          date(greg_year(2019), greg_month(1), greg_day(20)));
+      core::TimeStamp actual_exp = *securities::expiration(spread);
+      REQUIRE((actual_exp == core::TimeStamp{2019, 1, 20}));
      auto &leg = spread.spread().legs(1).security();
-      REQUIRE(
-          securities::expiration(leg) ==
-          date(greg_year(3020), greg_month(6), greg_day(19)));
+      actual_exp = *securities::expiration(leg);
+      REQUIRE((actual_exp == core::TimeStamp{3020, 6, 19}));
      REQUIRE(securities::is_expired(spread));
      REQUIRE(!securities::is_expired(leg));
   }

   SECTION("test_expiration_sod_boundary") {
-
-      using namespace boost::gregorian;
-
      auto fut = utils::pb::from_string<api::Security>(fut_text);
-      REQUIRE(securities::is_expired(
-          fut, datetime::make_ptime(2019, 1, 21, 0, 0, 0, 0), 16, 30));
-
-      REQUIRE(!securities::is_expired(
-          fut, datetime::make_ptime(2019, 1, 19, 0, 0, 0, 0), 16, 30));
-
-      REQUIRE(!securities::is_expired(
-          fut, datetime::make_ptime(2019, 1, 20, 16, 29, 0, 0), 16, 30));
-
-      REQUIRE(securities::is_expired(
-          fut, datetime::make_ptime(2019, 1, 20, 16, 31, 0, 0), 16, 30));
+      REQUIRE(securities::is_expired(fut, {2019, 1, 21}, 16, 30));
+      REQUIRE(!securities::is_expired(fut, {2019, 1, 19}, 16, 30));
+      REQUIRE(!securities::is_expired(fut, {2019, 1, 20, 16, 29, 0}, 16, 30));
+      REQUIRE(securities::is_expired(fut, {2019, 1, 20, 16, 31, 0}, 16, 30));
   }
 }

@@ -1,21 +0,0 @@
-#include "CurlHandle.h"
-#include <curl/curl.h>
-#include <stdexcept>
-
-namespace bts::http {
-
-CurlHandle::CurlHandle() {
-   const auto eCode = curl_global_init(CURL_GLOBAL_ALL);
-   if (eCode != CURLE_OK) {
-      throw std::runtime_error{"Error initializing libCURL"};
-   }
-}
-
-CurlHandle::~CurlHandle() { curl_global_cleanup(); }
-
-CurlHandle &CurlHandle::instance() {
-   static CurlHandle inst{};
-   return inst;
-}
-
-} // namespace bts::http
@@ -1,21 +0,0 @@
-#pragma once
-
-namespace bts::http {
-
-class CurlHandle {
- public:
-   static CurlHandle &instance();
-
-   CurlHandle(CurlHandle const &) = delete;
-   CurlHandle(CurlHandle &&) = delete;
-
-   CurlHandle &operator=(CurlHandle const &) = delete;
-   CurlHandle &operator=(CurlHandle &&) = delete;
-
-   ~CurlHandle();
-
- private:
-   CurlHandle();
-};
-
-} // namespace bts::http
@@ -1,55 +0,0 @@
-#include "HTTPClient.h"
-#include "HTTPRequest.h"
-#include "HTTPResponse.h"
-#include "lib/io/URL.h"
-#include "lib/logging/logging.h"
-#include <curl/curl.h>
-#include <fmt/format.h>
-
-namespace bts::io::http {
-
-static size_t write_data_(
-    void *ptr, size_t size, size_t nmemb, std::string *data) {
-   data->append((char *)ptr, size * nmemb);
-   return size * nmemb;
-}
-
-HTTPResponse HTTPClient::get(const HTTPRequest &request) {
-   auto *curl = curl_easy_init();
-   auto url_str = request.url().to_string();
-   // LOG(md, bts::Info, "do GET: {}", url_str);
-   curl_easy_setopt(curl, CURLOPT_URL, url_str.c_str());
-
-   struct curl_slist *request_headers = NULL;
-   for (const auto &[k, v] : request.headers()) {
-      request_headers = curl_slist_append(
-          request_headers, fmt::format("{}: {}", k, v).c_str());
-   }
-
-   std::string response_string;
-   std::string header_string;
-   std::string response_header_string;
-   curl_easy_setopt(curl, CURLOPT_WRITEFUNCTION, write_data_);
-   curl_easy_setopt(curl, CURLOPT_WRITEDATA, &response_string);
-   curl_easy_setopt(curl, CURLOPT_HEADERDATA, &header_string);
-
-   char *url;
-   long response_code;
-   double elapsed;
-   curl_easy_getinfo(curl, CURLINFO_RESPONSE_CODE, &response_code);
-   curl_easy_getinfo(curl, CURLINFO_TOTAL_TIME, &elapsed);
-   curl_easy_getinfo(curl, CURLINFO_EFFECTIVE_URL, &url);
-   // curl_easy_getinfo(curl, CURLINFO_HEADER_OUT, &response_header_string);
-
-   curl_easy_perform(curl);
-   curl_easy_cleanup(curl);
-   curl = NULL;
-   HTTPResponse response;
-   response.headers() = HTTPHeaders(response_header_string);
-   response.body() = std::move(response_string);
-   response.code() = response_code;
-   // LOG(md, bts::Info, "do GET: {}.  Received response:\n{}", url_str, response);
-   return response;
-}
-
-} // namespace bts::io::http
@@ -1,35 +0,0 @@
-#include <algorithm>
-#include <atomic>
-#include <cstddef> // std::size_t
-#include <cstdio>  // snprintf
-#include <cstdlib>
-#include <cstring> // strerror, strlen, memcpy, strcpy
-#include <ctime>
-#include <curl/curl.h>
-#include <fstream>
-#include <functional>
-#include <iostream>
-#include <memory> // std::unique_ptr
-#include <mutex>
-#include <sstream>
-#include <stdarg.h> // va_start, etc.
-#include <stdio.h>
-#include <stdlib.h>
-#include <string>
-#include <sys/stat.h>
-#include <sys/types.h>
-#include <unordered_map>
-#include <vector>
-
-namespace bts::io::http {
-
-class HTTPRequest;
-class HTTPResponse;
-
-class HTTPClient {
- public:
-   // HTTPClient(const std::string &url);
-   HTTPResponse get(const HTTPRequest &request);
-};
-
-} // namespace bts::io::http
@@ -1,17 +0,0 @@
-#include "HTTPRequest.h"
-
-namespace bts::io::http {
-
-io::URL &HTTPRequest::url() { return url_; }
-
-const io::URL &HTTPRequest::url() const { return url_; }
-
-HTTPHeaders &HTTPRequest::headers() { return headers_; }
-
-const HTTPHeaders &HTTPRequest::headers() const { return headers_; }
-
-std::string &HTTPRequest::body() { return body_; }
-
-const std::string &HTTPRequest::body() const { return body_; }
-
-} // namespace bts::io::http
@@ -1,26 +0,0 @@
-#pragma once
-#include "HTTPHeaders.h"
-#include "lib/io/URL.h"
-#include <string>
-#include <vector>
-
-namespace bts::io::http {
-
-class HTTPRequest {
- public:
-   io::URL &url();
-   const io::URL &url() const;
-
-   HTTPHeaders &headers();
-   const HTTPHeaders &headers() const;
-
-   std::string &body();
-   const std::string &body() const;
-
- private:
-   io::URL url_;
-   HTTPHeaders headers_;
-   std::string body_;
-};
-
-} // namespace bts::io::http
@@ -14,7 +14,9 @@ namespace bts::io::http {
 #define FOR_ALL_VALUES(per_value)                                            \
   per_value(HttpMethod, NONE)                                       \
   per_value(HttpMethod, GET)                                        \
-   per_value(HttpMethod, POST)
+   per_value(HttpMethod, POST)                                       \
+   per_value(HttpMethod, PUT)                                        \
+   per_value(HttpMethod, DELETE)
 // clang-format on

 /**
@@ -0,0 +1,38 @@
+#pragma once
+
+#include <cstddef>
+#include <curl/curl.h>
+#include <string>
+#include <unordered_map>
+
+namespace bts::http {
+
+class CurlHeaders {
+ public:
+   CurlHeaders() = default;
+
+   explicit CurlHeaders(
+       const std::unordered_map<std::string, std::string> &items) {
+      for (const auto &[k, v] : items) {
+         append(k, v);
+      }
+   }
+
+   ~CurlHeaders() {
+      if (data_) {
+         curl_slist_free_all(data_);
+      }
+   }
+
+   void append(const std::string &key, const std::string &value) {
+      const std::string str = key + ": " + value;
+      data_ = curl_slist_append(data_, str.c_str());
+   }
+
+   curl_slist *data() { return data_; }
+
+ private:
+   curl_slist *data_ = NULL;
+};
+
+} // namespace bts::http
@@ -0,0 +1,28 @@
+#pragma once
+
+#include <atomic>
+#include <curl/curl.h>
+#include <curl/easy.h>
+
+namespace bts::http {
+
+struct CurlSession {
+   static inline std::atomic<int> ref_count = 0;
+   CURL *curl = nullptr;
+
+   CurlSession() {
+      if (++ref_count == 1) {
+         curl_global_init(CURL_GLOBAL_ALL);
+      }
+      curl = curl_easy_init();
+   }
+
+   ~CurlSession() {
+      curl_easy_cleanup(curl);
+      if (--ref_count == 0) {
+         curl_global_cleanup();
+      }
+   }
+};
+
+} // namespace bts::http
@@ -0,0 +1,167 @@
+#include "http_client.h"
+#include "lib/io/JsonDumper.h"
+#include "lib/io/http/HttpMethod.h"
+#include "lib/io/http/curl_headers.h"
+#include "lib/io/http/curl_session.h"
+#include "lib/io/http/http_request.h"
+#include "lib/io/http/http_response.h"
+#include "lib/io/http/i_http_response.h"
+#include "lib/io/http/query_params.h"
+#include "lib/logging/logging.h"
+
+#include <curl/curl.h>
+#include <curl/easy.h>
+#include <string>
+
+namespace bts::http {
+
+using bts::io::JsonDumper;
+using nlohmann::json;
+using std::exception;
+using std::int64_t;
+using std::pair;
+using std::string;
+using std::tuple;
+using std::unordered_map;
+using std::vector;
+
+static size_t write_data(
+    void *ptr, size_t size, size_t nmemb, IHttpResponse *data) {
+   return data->write((char *)ptr, size, nmemb);
+}
+
+static size_t write_headers(
+    void *ptr, size_t size, size_t nmemb, IHttpResponse *data) {
+   return data->write_headers((char *)ptr, size, nmemb);
+}
+
+void HttpClient::get(
+    string url,
+    unordered_map<string, string> headers,
+    IHttpResponse *response) {
+   HttpRequest req;
+   req.url = std::move(url);
+   req.headers = std::move(headers);
+   req.method = io::http::HttpMethod::GET;
+   send(req, response);
+}
+
+void HttpClient::post(
+    string url,
+    string body,
+    unordered_map<string, string> headers,
+    IHttpResponse *response) {
+   HttpRequest req;
+   req.url = std::move(url);
+   req.body = std::move(body);
+   req.headers = std::move(headers);
+   req.method = io::http::HttpMethod::POST;
+   send(req, response);
+}
+
+void HttpClient::put(
+    string url,
+    string body,
+    unordered_map<string, string> headers,
+    IHttpResponse *response) {
+   HttpRequest req;
+   req.url = std::move(url);
+   req.body = std::move(body);
+   req.headers = std::move(headers);
+   req.method = io::http::HttpMethod::PUT;
+   send(req, response);
+}
+
+void HttpClient::del(
+    string url,
+    string body,
+    unordered_map<string, string> headers,
+    IHttpResponse *response) {
+   HttpRequest req;
+   req.url = std::move(url);
+   req.body = std::move(body);
+   req.headers = std::move(headers);
+   req.method = io::http::HttpMethod::DELETE;
+   send(req, response);
+}
+
+void HttpClient::send(const HttpRequest &req, IHttpResponse *response) {
+   LOG(Http, Info, "HttpClient::send. {}", JsonDumper::dump(req));
+   CurlSession session;
+   CURL *curl = session.curl;
+   CurlHeaders curl_headers{req.headers};
+   // Prepare req
+   curl_easy_setopt(curl, CURLOPT_FAILONERROR, true);
+   curl_easy_setopt(curl, CURLOPT_URL, req.url.c_str());
+   curl_easy_setopt(curl, CURLOPT_HTTPHEADER, curl_headers.data());
+   if (response != nullptr) {
+      curl_easy_setopt(curl, CURLOPT_HEADERFUNCTION, write_headers);
+      curl_easy_setopt(curl, CURLOPT_HEADERDATA, response);
+      curl_easy_setopt(curl, CURLOPT_WRITEFUNCTION, write_data);
+      curl_easy_setopt(curl, CURLOPT_WRITEDATA, response);
+   }
+   switch (req.method.value) {
+   case io::http::HttpMethod::GET:
+      curl_easy_setopt(curl, CURLOPT_HTTPGET, 1L);
+      break;
+   case io::http::HttpMethod::POST:
+      curl_easy_setopt(curl, CURLOPT_HTTPPOST, 1L);
+      curl_easy_setopt(curl, CURLOPT_POSTFIELDS, req.body.c_str());
+      curl_easy_setopt(curl, CURLOPT_POSTFIELDSIZE, req.body.size());
+      break;
+   case io::http::HttpMethod::PUT:
+      curl_easy_setopt(curl, CURLOPT_CUSTOMREQUEST, "PUT");
+      curl_easy_setopt(curl, CURLOPT_POSTFIELDS, req.body.c_str());
+      curl_easy_setopt(curl, CURLOPT_VERBOSE, 1L);
+      // curl_easy_setopt(curl, CURLOPT_POSTFIELDSIZE, req.body.size());
+      break;
+   case io::http::HttpMethod::DELETE:
+      curl_easy_setopt(curl, CURLOPT_CUSTOMREQUEST, "DELETE");
+      curl_easy_setopt(curl, CURLOPT_POSTFIELDS, req.body.c_str());
+      break;
+   case io::http::HttpMethod::NONE:
+   default:
+      break;
+   }
+   // Send it
+   CURLcode curl_code = curl_easy_perform(curl);
+   long response_code;
+   double elapsed;
+   char *effective_url = NULL;
+   curl_easy_getinfo(curl, CURLINFO_RESPONSE_CODE, &response_code);
+   curl_easy_getinfo(curl, CURLINFO_TOTAL_TIME, &elapsed);
+   curl_easy_getinfo(curl, CURLINFO_EFFECTIVE_URL, &effective_url);
+   const char *err = curl_easy_strerror(curl_code);
+   try {
+      response->write_result(response_code, string{err}, string{effective_url});
+   } catch (const exception &ex) {
+      LOG(Http, Error, "HttpClient::perform. IHttpResponse threw: {}",
+          string{ex.what()});
+   } catch (...) {
+      LOG(Http, Error, "HttpClient::perform. IHttpResponse threw unknown ex.");
+   }
+}
+
+tuple<string, string, int64_t> HttpClient::get_oauth_token(
+    const string &url, const string &client_id, const string &client_secret) {
+   LOG(Gen, Info, "Requesting oauth token from {}", url);
+   vector<pair<string, string>> query_params{
+       {"grant_type", "client_credentials"},
+       {"client_id", client_id},
+       {"client_secret", client_secret}};
+   HttpResponse res;
+   HttpClient::post(url, query_params_string(query_params), {}, &res);
+   LOG(Gen, Info, "Received oauth token:\n{}", JsonDumper::dump(res));
+   try {
+      const auto j = json::parse(res.body);
+      return {
+          j.at("access_token").get<string>(), j.at("token_type").get<string>(),
+          j.at("expires_in").get<int64_t>()};
+   } catch (const exception &e) {
+      LOG(Gen, Error, "Failed to parse oauth response json.{}\n{}",
+          string{e.what()}, res.body);
+   }
+   return {};
+}
+
+} // namespace bts::http
@@ -0,0 +1,63 @@
+#pragma once
+
+#include "lib/io/http/http_request.h"
+#include "lib/io/http/i_http_response.h"
+
+#include <cstdint>
+#include <string>
+#include <tuple>
+#include <unordered_map>
+
+namespace bts::http {
+
+class HttpClient {
+ public:
+   /**
+    * Performs an http GET request
+    */
+   static void get(
+       std::string url,
+       std::unordered_map<std::string, std::string> headers,
+       IHttpResponse *response);
+
+   /**
+    * Performs an http POST request
+    */
+   static void post(
+       std::string url,
+       std::string body,
+       std::unordered_map<std::string, std::string> headers,
+       IHttpResponse *response);
+
+   /**
+    * Performs an http PUT request
+    */
+   static void put(
+       std::string url,
+       std::string body,
+       std::unordered_map<std::string, std::string> headers,
+       IHttpResponse *response);
+
+   /**
+    * Performs an http DELETE request
+    */
+   static void del(
+       std::string url,
+       std::string body,
+       std::unordered_map<std::string, std::string> headers,
+       IHttpResponse *response);
+
+   /**
+    * Requests an oauth access token
+    *
+    * @return tuple of: access token, token type, and seconds until expiration
+    */
+   static std::tuple<std::string, std::string, std::int64_t> get_oauth_token(
+       const std::string &url,
+       const std::string &client_id,
+       const std::string &client_secret);
+
+   static void send(const HttpRequest &req, IHttpResponse *res);
+};
+
+} // namespace bts::http
Author	SHA1	Message	Date
John Colvin	9deb8cc237	fix build: Trivial changes to fix emumba_deribit build * Use "Unix Makefiles" generator when building websocket-library. - Ninja tries to build things in parallel, and doesn't build the ExternalProject dependencies in the right order. Consequently, the websocket-library build fails because it can't find "libspdlog.a". * Replace a few leftover SPDLOG_ERROR macros w/ LOG_ERROR (the latter logs in our system, not spdlog).	2025-01-10 19:36:18 +00:00
Mohammad Arham	11ff46a056	scafolding	2025-01-09 19:56:54 +00:00
John Colvin	ba4f8f67e3	fix: Disable binance usds futs symbol filter config This was a premature optimization, and has become a burden now that we need to subscribe to large symbol sets for Alphaworx.	2025-01-06 05:00:04 +00:00
John Colvin	9c78380490	fix build (gcc rel): Don't fallthrough switch cases	2025-01-06 03:52:25 +00:00
John Colvin	f6166f0674	fix: Show iceberg show qty in algo params grid Bugs (fixed): * In the ladder, you can't view/edit the iceberg show quantity. Cause: No view template was defined for nullable double params. Most likely, we changed the show qty in the vm from double to nullable double at some point, and forgot to reflect this change in the view. An easy mistake to make as it would not have broken the build.	2025-01-05 15:58:29 -05:00
John Colvin	6334a902e7	fix: Reclaim keyboard focus when mview popup is clicked	2025-01-05 15:29:11 -05:00
John Colvin	082ca62fff	fix: Focus price textbox when mview order ticket popup opens	2025-01-05 15:18:46 -05:00
John Colvin	9dbd8dd7c6	fix: Set focus through FocusScope element if one exists	2025-01-05 15:18:46 -05:00
John Colvin	8f7a4d0df6	Fix typo in AutoFocusTargetProvider dependency prop	2025-01-05 15:18:46 -05:00
John Colvin	f372dd56b3	fix: Return synthetic name in get_security_desc If the security is a synthetic (of course). Bugs (fixed): * Changing limits in the risk ui causes sparkd to assert crash	2025-01-05 20:15:10 +00:00
John Colvin	86eba20d61	build: Make compiler/build type case insensitive	2025-01-05 06:25:11 +00:00
John Colvin	35a985280d	fix: Run heartbeat timer in MarketDataGateway This is why we were getting those heartbeat asserts. Previously, only ExchangeSimApplication would start this timer. Since MarketDataGateway doesn't share its MDProducer with ExchangeSimApplication, it must start one itself.	2025-01-03 16:59:07 +00:00
John Colvin	f5e1b3319c	cleanup: Don't write to cout in SparkDataDirectory Don't need that noise.	2025-01-03 16:59:07 +00:00
John Colvin	65c3948e4f	cleanup: Remove unused code	2025-01-03 16:59:07 +00:00
John Colvin	002bdff9a3	fix: Don't parse non-existent json files in SparkDataDirectory Bugs (fixed): * Sparkd assert crashes if binance_usds_futures_execution.json file doesn't exist	2025-01-03 16:59:07 +00:00
John Colvin	bb6e5dd548	diag: Fix sparkd log path in show_last_run.sh	2025-01-03 16:59:07 +00:00
John Colvin	7c1aed3339	fix build (api server): Remove old cpps from source file list	2025-01-03 16:59:07 +00:00
John Colvin	22ae51fb99	build: Clean up build py scripts	2025-01-03 16:59:07 +00:00
John Colvin	a53d347f81	fix build (warns): Update cmake min version to 3.10 Versions < 3.10 will soon be deprecated (we just started getting warnings about it).	2025-01-03 16:59:07 +00:00
John Colvin	838dbcd886	build: In py build scripts, exit with code 1 on failure Previously, they would exit with 0 no matter what happened.	2025-01-03 16:59:07 +00:00
John Colvin	5ed2309d22	fix: Add switch case for VENUE_BTS in VenueIndex	2025-01-03 16:59:07 +00:00
John Colvin	4d41704dd2	build: Configure admin webapp build entirely in CMakeLists Get rid of build_admin_web_app.py	2025-01-03 16:59:07 +00:00
John Colvin	bd1000e518	build: Get rid of explicit "targets" in CMakePresets for sparkd So we don't have to worry about keeping it synchronized with the actual targets in the build (just one more thing that can break). You can build specific targets via the cmake cli anyways.	2025-01-03 16:59:07 +00:00
John Colvin	dd4aa3c2ed	build: Update fable in dotnet-tools.json	2025-01-03 16:59:07 +00:00
John Colvin	fee9d3f597	build: Update webpack-cli in admin webapp	2025-01-03 16:59:07 +00:00
John Colvin	416f9aef31	vscode: Add Sparkd Test to launch.json	2025-01-03 16:59:07 +00:00
John Colvin	48f5e4c201	vscode: Fix sparkd launch.json	2025-01-03 16:59:07 +00:00
John Colvin	4438b4ca8d	feat: Support fractional quantities in synthetic book	2025-01-03 16:59:07 +00:00
John Colvin	73356e0d61	cli: Add cmd to print security info from the db to console Usage: sparkd md dump --db	2025-01-03 16:59:07 +00:00
John Colvin	4a285fa581	cleanup: Make fn const	2025-01-03 16:59:07 +00:00
John Colvin	cfa82c2177	cleanup: Remove erroneous fwd declared class	2025-01-03 16:59:07 +00:00
John Colvin	0410a0431a	fix: Encode exp in valid date int when storing in sqlite The strange bit shifting we were doing before doesn't support large dates. Bugs (fixed): * Synthetics with Binance perpetual future legs don't appear on the front end after sparkd restarts. * Cause: Exp dates for Binance perpetual futures aren't properly encoded in the database. As a result, when loaded from disk, they are mistakenly identified as expired, along with any synthetics that have them as legs.	2025-01-03 16:59:07 +00:00
John Colvin	02843f14ff	cleanup: Get rid of trivial fn that's only used in one place Move the logic into the function that used to call it.	2025-01-03 16:59:07 +00:00
John Colvin	f1f1d4259c	diag: Add logs around security discovery in MDConsumer This is multi-threaded event driven code. Without descriptive logs in the right places, diagnosing problems is difficult and tedious.	2025-01-03 16:59:07 +00:00
John Colvin	71fc43af68	cleanup: Flatten nested if blocks with continue statements Easier to read	2025-01-03 16:59:07 +00:00
John Colvin	350144ffd5	fix: Use security desc to get unique id (along with venue, type) I'm guessing this typo went unnoticed because it didn't break the build (another arg of the same was being used instead).	2025-01-03 16:59:07 +00:00
John Colvin	da227a5550	Support synths at Binance, Binance USDS Futures, Deribit, and Okx	2025-01-03 16:59:07 +00:00
John Colvin	0c92cf3a87	Add double ops to IntPrice	2025-01-03 16:59:07 +00:00
John Colvin	1188af391e	Add trivial fns to TimeStamp for convenience * += and -= operators * 'date' fn * 'to_date_int' fn	2025-01-03 16:59:07 +00:00
John Colvin	0bb85cf915	refactor: Change month and day params from uint32 to uint8 They only need to hold values 1-12 and 1-31, respectively. A single byte is more than sufficient. Also add ymdhms TimeStamp constructor.	2025-01-03 16:59:07 +00:00
John Colvin	d77f5996fe	cleanup: Make a couple TimeStamp fns const	2025-01-03 16:59:07 +00:00
John Colvin	940c41a274	Add int32 arithmetic operators to domain::Price	2025-01-03 16:59:07 +00:00
John Colvin	e671960dc3	Add double arithmetic operators to domain::Price	2025-01-03 16:59:07 +00:00
John Colvin	8fa34b6b39	fix: Multiply additive factors by exponent in domain::Price We must not have been using these operators because the implementation was very broken (yet it doesn't seemed to have caused any bugs).	2025-01-03 16:59:07 +00:00
John Colvin	f9d5429b5b	cleanup: Remove unused code	2025-01-03 16:53:05 +00:00
Mohammad Arham	2c342482eb	Merge branch 'development.crypto' of bitbucket.org:bluetradesys/spark into development.crypto	2025-01-02 15:30:13 -05:00
Mohammad Arham	3aefac26c2	fix: Timer condition was incorrect	2025-01-02 20:15:38 +00:00
John Colvin	1554594cdd	diag: Throttle noisy log in Ladder	2025-01-02 12:15:22 -05:00
John Colvin	727a0f38b5	fix: Prevent arithmetic overflow when converting long to int in Ladder	2025-01-02 12:13:58 -05:00
John Colvin	7a0e8cdb26	fix: Properly format spread/synth leg qty in generated desc	2025-01-01 13:12:17 -05:00
John Colvin	a4e97b04d7	fix: Don't propagate security criteria changes to other windows Bugs (fixed): * Security criteria changes in mview and time/sales propagate to other windows of the same type.	2024-12-31 11:06:24 -05:00
John Colvin	463980a6ef	Fix typo in get_binance_md.py	2024-12-29 18:37:54 +00:00
John Colvin	9b25857042	fix: Ensure "earliest_expiration" fn actually returns min exp Previously, it assumed the first leg would have the earliest exp. While this may be true for most securities, it isn't guarenteed by our data model.	2024-12-29 13:26:44 -05:00
John Colvin	ef19e7d76b	fix: Don't assume all securities have expirations in pnl vm Bugs (fixed): * PNL window crashes if data has Spots (i.e., securities without exps)	2024-12-29 13:17:17 -05:00
Taha Afzal	471faec2cb	Added more etsts for fractional qtys	2024-12-28 08:31:16 +00:00
Taha Afzal	ce96d42db8	Merging	2024-12-26 07:01:30 +00:00
Mohammad Arham	187894ca84	FIX: makes gateway factory work	2024-12-24 17:56:49 +00:00
John Colvin	63492c4575	fix: Remove erroneous assert in LadderRender	2024-12-21 23:24:52 -05:00
John Colvin	e150c611f6	fix: Round fractional order qty to 6 decimal places It was previously rounding to 2 decimal places, which isn't enough for many crypto currencies (e.g., BTCUSDT at Binance)	2024-12-21 23:17:52 -05:00
John Colvin	bb837d6c26	fix build: Trivial changes in DatabentoDataHandler	2024-12-22 04:00:23 +00:00
John Colvin	965e541ac8	Fix json structures and endpoints for Binance USDS Futures md	2024-12-22 04:00:23 +00:00
John Colvin	1b232aa610	feat: Add Binance USDS Futures execution	2024-12-22 04:00:23 +00:00
John Colvin	25439b094a	improvement: Perform sec lookup in const time	2024-12-21 20:08:19 +00:00
John Colvin	f927ef2c5e	Add conf_json dir to spark data file tree So we don't have to add proto files to the build for trivial config.	2024-12-21 20:08:19 +00:00
John Colvin	d42389b75a	Don't needlessly download cme config xml When we aren't subscribing to cme md (i.e., no glbx symbols in spark conf).	2024-12-21 20:08:19 +00:00
John Colvin	671cbcf03a	Fix bts::Quantity string fmt so it includes fractional qty	2024-12-21 20:01:55 +00:00
John Colvin	e589e23f41	Add py script to get md at binance For quick/easy troubleshooting (much easier than doing stuff in c++)	2024-12-21 20:01:55 +00:00
John Colvin	11fadc0a3e	Expose "run" fn in WebsocketClient So it doesn't have to be hidden in a background thread	2024-12-21 20:01:55 +00:00
John Colvin	b8c2d93866	Add ping/pong facilities to WebsocketClient	2024-12-21 20:01:55 +00:00
John Colvin	67a23dd196	Add open/close callbacks to WebsocketClient	2024-12-21 20:01:55 +00:00
John Colvin	ff22c4eb5f	diag: Change WebsocketClient log category to "Http" A better fit than "Gen"	2024-12-21 20:01:55 +00:00
John Colvin	5addff4c50	cleanup: Remove redundant fn HttpRequest::perform This was doing the same thing as HttpClient::send. I likely refactored this function during development, then forgot to remove it.	2024-12-21 20:01:55 +00:00
John Colvin	142e1bd2da	Add HttpMethod "DELETE" In Binance's REST api, orders are canceled via DELETE requests.	2024-12-21 20:01:55 +00:00
John Colvin	d05d91bbe7	diag: Log Http requests	2024-12-21 20:01:55 +00:00
Mohammad Arham	3a90ca9cb5	Add: Johns changes to sec oracle	2024-12-17 14:47:37 +00:00
John Colvin	85a46727be	Improve c++ http client facilities	2024-12-11 19:28:36 +00:00
Mohammad Arham	624fd2c976	FIX: BUILD Error	2024-12-09 01:34:29 +00:00
Mohammad Arham	16d599d2d6	Add: OKX execution gateway connected	2024-12-08 20:14:16 -05:00
John Colvin	c00e4fcb6b	feat: Add Binance Coin Futures market data	2024-12-06 00:39:41 +00:00
John Colvin	03e19afe1d	Fix qty display conf for BTC/ETH futs at Binance Need to show 3 decimal places (qty tick size is 0.001)	2024-12-05 21:05:53 +00:00
John Colvin	058e11eadc	fix build: Fix install path for sparkd	2024-12-05 18:31:36 +00:00
John Colvin	6ff52e7107	fix build: Add onixs libs again Taha accidentally reverted externals	2024-12-04 16:56:46 +00:00
John Colvin	ee4c28ed41	Port ws client over from cme-websocket-connector Need this to implement execution at crypto exchanges (beginning with Binance and OKX)	2024-12-04 16:56:46 +00:00
Mohammad Arham	50d26f0060	Merge branch 'development' of bitbucket.org:bluetradesys/spark into development	2024-12-03 09:52:09 -05:00
Mohammad Arham	36378e6074	Add:: Recording md to sql	2024-12-03 14:51:32 +00:00
John Colvin	8cde8ee6e1	build: Add script to publish the api server This will build the app as a self-contained executable here: build/Release/bin/net7.0/linux-x64/publish	2024-11-28 16:35:21 +00:00
John Colvin	4f8fad444e	fix build: Add missing csproj deps to Spark.Api.Server.sln	2024-11-28 16:35:20 +00:00
John Colvin	832c645a30	build: Map cmake install dir to associated preset Cpp bins are now installed to "install/<cmake_preset>/bin". For example, "install/sparkd-release-gcc/bin", "install/spark-api-server-linux/bin", etc.	2024-11-28 16:35:20 +00:00
John Colvin	691343d9b6	feat: Add datastreams to spark api server	2024-11-28 06:14:00 +00:00
Taha	461cbad338	Adding FakeBBO stream	2024-11-27 13:36:14 -05:00
Taha	fecbcbb6b6	Merge branch 'development' of bitbucket.org:bluetradesys/spark into development	2024-11-26 22:33:31 -05:00
Taha	aab8e83543	Some rewrite of streams that fetch bars, and making the other business streams thin wrapper over stream primitives	2024-11-26 22:33:26 -05:00
Mohammad Arham	420b46741e	ADD: Config for sql writer in mdrecorder	2024-11-24 19:14:45 +00:00
Mohammad Arham	e8f65f95bd	Wrapped proto write in function in mdrecorder	2024-11-24 19:13:36 +00:00