Add PUT method to http client

fix ws client
refactor: Move OrderServiceServer to sparkd
2024-12-04 20:22:38 +00:00 · 2024-12-04 20:22:13 +00:00 · 2024-11-30 17:30:10 +00:00 · 2024-11-29 15:05:54 +00:00 · 2024-11-29 15:05:54 +00:00 · 2024-11-29 15:05:54 +00:00
354 changed files with 9951 additions and 1240 deletions
@@ -25,6 +25,12 @@
      "commands": [
        "fake"
      ]
+    },
+    "dotnet-t4": {
+      "version": "3.0.0",
+      "commands": [
+        "t4"
+      ]
    }
  }
-}
+}
@@ -0,0 +1,81 @@
+# This file defines the global default configuration for docker deployments.
+# It is tracked in source control. For temporary modifications, please create
+# a separate file called ".env.local" (it's in .gitignore, so won't be tracked).
+# If a variable is defined in both files, the value in .env.local will be used.
+#
+# About the deployment procedure:
+#   We use Docker Compose to build images and launch containers for various services
+#   that make up our software suite:
+#     - sparkd
+#     - pubsub
+#     - md-publisher
+#     - webapp (DSQA on the web)
+#     - api-server (websocket based)
+#   From inside a devcontainer, you can launch all services in separate containers
+#   on your machine by running:
+#
+#     tools/deploy.sh
+#
+#   This script builds release versions for each service, packaged (generally) in
+#   self-contained directories. Next, it runs a few "docker compose" commands to
+#   build docker images loaded with the self-contained binaries from the previous
+#   step, and start containers with those images to run each service. To view the
+#   commands that deploy.sh runs internally (without actually running them), run:
+#
+#     tools/deploy.sh --dry-run
+#
+#   For additional information/help, run:
+#
+#     tools/deploy.sh --help
+#
+
+### Sparkd ###
+# The internal image for sparkd to use. For more info, view:
+#   src/app_sparkd/Dockerfile
+SPARKD__IMAGE=sparkd
+# The 'spark.conf' for this service to use
+SPARKD__CONF_FILE=src/app_sparkd/spark.conf
+# To run replay md, put recorded md files in SPARKD__MD_RECORDING_DIR, and use:
+#   SPARKD__IMAGE=sparkd_md_replay
+#   SPARKD__CONF_FILE=src/app_sparkd/spark.local.conf
+# (configure which md to replay in the new SPARKD__CONF_FILE)
+SPARKD__MD_RECORDING_DIR=.md_recording
+SPARKD__SOURCE_DIR=install/sparkd-release-gcc
+SPARKD__CONFIG_PORT=8000
+SPARKD__QUICKBUS_PORT=8001
+SPARKD__SIM_FIX_PORT=8002
+
+### Pubsub ###
+# The 'spark.conf' for this service to use
+PUBSUB__CONF_FILE=src/DataStreamSuite/PubSubServer/spark.conf
+PUBSUB__CS_SOURCE_DIR=build/Release/bin/net7.0/linux-x64/publish
+PUBSUB__CPP_SOURCE_DIR=install/data-stream-suite-linux
+# The appsettings to load at runtime. Expects a file named:
+#   src/DataStreamSuite/PubSub/appsettings.${PUBSUB__DOTNET_ENVIRONMENT}.json
+PUBSUB__DOTNET_ENVIRONMENT=Staging
+
+### Md Publisher ###
+# The 'spark.conf' for this service to use
+MD_PUBLISHER__CONF_FILE=src/DataStreamSuite/DataStreamPublisher/spark.conf
+MD_PUBLISHER__SOURCE_DIR=build/Release/bin/net7.0/linux-x64/publish
+# Determines which appsettings to load at runtime. Expects a file named:
+#   src/DataStreamSuite/DataStreamPublisher/appsettings.${MD_PUBLISHER__DOTNET_ENVIRONMENT}.json
+MD_PUBLISHER__DOTNET_ENVIRONMENT=MarketData
+
+### Webapp (DSQA on the web) ###
+# The 'spark.conf' for this service to use
+WEBAPP__CONF_FILE=src/client/spark_api_server/spark.conf
+WEBAPP__SOURCE_DIR=build/Release/bin/net7.0/linux-x64/publish
+WEBAPP__WEB_ROOT=build/Bts.DataStreamWebApp.Client/production/dist
+# Determines which appsettings to load at runtime. Expects a file named:
+#   src/DataStreamSuite/Bts.DataStreamWebApp.Server/appsettings.${WEBAPP__ASPNETCORE_ENVIRONMENT}.json
+WEBAPP__ASPNETCORE_ENVIRONMENT=Staging
+
+### Api Server ###
+# The 'spark.conf' for this service to use
+API_SERVER__CONF_FILE=src/client/spark_api_server/spark.conf
+API_SERVER__CS_SOURCE_DIR=build/Release/bin/net7.0/linux-x64/publish
+API_SERVER__CPP_SOURCE_DIR=install/spark-api-server-linux
+# Determines which appsettings to load at runtime. Expects a file named:
+#   src/client/spark_api_server/appsettings.${API_SERVER__ASPNETCORE_ENVIRONMENT}.json
+API_SERVER__ASPNETCORE_ENVIRONMENT=Staging
@@ -59,3 +59,6 @@ trace_data/
 .devcontainer/
 *.Generated.cs
 *.lst
+src/app_sparkd/spark.local.conf
+.env.local
+.md_recording
@@ -62,7 +62,7 @@
      "name": "Bts.DataStreamWebApp.Server",
      "type": "coreclr",
      "request": "launch",
-      "preLaunchTask": "Build DataStream WebApp (Debug)",
+      "preLaunchTask": "Build DataStream (Debug)",
      "program": "${workspaceFolder}/build/Debug/bin/net7.0/Bts.DataStreamWebApp.Server.dll",
      "cwd": "${workspaceFolder}/build/Debug/bin/net7.0",
      "env": {
@@ -14,16 +14,36 @@ Project("{FAE04EC0-301F-11D3-BF4B-00C04F79EFBC}") = "spark_test", "src\client\sp
 EndProject
 Project("{FAE04EC0-301F-11D3-BF4B-00C04F79EFBC}") = "Spark.Console", "src\client\Spark.Console\Spark.Console.csproj", "{89D666E6-B500-498F-B026-4D167F68ADFC}"
 EndProject
+Project("{2150E333-8FDC-42A3-9474-1A3956D46DE8}") = "src", "src", "{267F81EF-A8AB-49FD-9F56-D7EC3C1F7C7C}"
+EndProject
+Project("{2150E333-8FDC-42A3-9474-1A3956D46DE8}") = "DataStreamSuite", "DataStreamSuite", "{C09ED521-D2DF-40BC-A76F-EB3CB870B2F6}"
+EndProject
+Project("{FAE04EC0-301F-11D3-BF4B-00C04F79EFBC}") = "DataStreamCore", "src\DataStreamSuite\DataStreamCore\DataStreamCore.csproj", "{EF26A475-6101-4C9C-A717-D60BB92A2302}"
+EndProject
+Project("{FAE04EC0-301F-11D3-BF4B-00C04F79EFBC}") = "SharedStreams", "src\DataStreamSuite\SharedStreams\SharedStreams.csproj", "{911B08A3-515A-441D-84C8-8A43740B5FC1}"
+EndProject
+Project("{FAE04EC0-301F-11D3-BF4B-00C04F79EFBC}") = "PubSub", "src\DataStreamSuite\PubSub\PubSub.csproj", "{14140310-2233-430A-821B-F3058A25234B}"
+EndProject
+Project("{FAE04EC0-301F-11D3-BF4B-00C04F79EFBC}") = "BaseUtilities", "src\DataStreamSuite\BaseUtilities\BaseUtilities.csproj", "{43146EA8-6FCC-46B8-88B0-42F9BB5E6E0A}"
+EndProject
+Project("{FAE04EC0-301F-11D3-BF4B-00C04F79EFBC}") = "ExpressionCore", "src\DataStreamSuite\ExpressionCore\ExpressionCore.csproj", "{6F22DB96-6EAB-4034-99D9-287021C36F01}"
+EndProject
+Project("{FAE04EC0-301F-11D3-BF4B-00C04F79EFBC}") = "ArithmeticExpressions", "src\DataStreamSuite\ArithmeticExpressions\ArithmeticExpressions.csproj", "{A5C33956-A41F-49B3-A41D-1082CBA59D02}"
+EndProject
+Project("{FAE04EC0-301F-11D3-BF4B-00C04F79EFBC}") = "DataStreamExpressions", "src\DataStreamSuite\DataStreamExpressions\DataStreamExpressions.csproj", "{2FE745C3-1EDF-4421-8CD7-A63EE0C1AE43}"
+EndProject
+Project("{FAE04EC0-301F-11D3-BF4B-00C04F79EFBC}") = "Bts.ColumnSortExpressions", "src\DataStreamSuite\Bts.ColumnSortExpressions\Bts.ColumnSortExpressions.csproj", "{FDA791E0-2B91-4AD5-B91D-CF920A4C4E38}"
+EndProject
+Project("{FAE04EC0-301F-11D3-BF4B-00C04F79EFBC}") = "ColumnMapExpressions", "src\DataStreamSuite\ColumnMapExpressions\ColumnMapExpressions.csproj", "{B945A183-2174-4ACB-B289-8A5C546656FE}"
+EndProject
+Project("{FAE04EC0-301F-11D3-BF4B-00C04F79EFBC}") = "BooleanExpressions", "src\DataStreamSuite\BooleanExpressions\BooleanExpressions.csproj", "{9F6D7DE8-185B-4C1D-B809-3314D092DF28}"
+EndProject
+Project("{FAE04EC0-301F-11D3-BF4B-00C04F79EFBC}") = "Bts.ColumnExpressions", "src\DataStreamSuite\Bts.ColumnExpressions\Bts.ColumnExpressions.csproj", "{6BD3F6D8-B6FC-44E8-B8FE-06352F638B9B}"
+EndProject
+Project("{FAE04EC0-301F-11D3-BF4B-00C04F79EFBC}") = "DataStream", "src\DataStreamSuite\DataStream\DataStream.csproj", "{39AEA075-6457-46C0-8821-1158BE6EFA57}"
+EndProject
 Project("{FAE04EC0-301F-11D3-BF4B-00C04F79EFBC}") = "Bts.Net", "src\DataStreamSuite\Bts.Net\Bts.Net.csproj", "{CCDCB00A-CABF-4D82-8A5F-33EEC8ACE51D}"
 EndProject
-Project("{FAE04EC0-301F-11D3-BF4B-00C04F79EFBC}") = "DataStreamCore", "src\DataStreamSuite\DataStreamCore\DataStreamCore.csproj", "{CB62D5D2-3495-4488-BFBC-7C7A3725A1EE}"
-EndProject
-Project("{FAE04EC0-301F-11D3-BF4B-00C04F79EFBC}") = "DataStream", "src\DataStreamSuite\DataStream\DataStream.csproj", "{05995ED5-650A-4955-9DFC-3242B737E77A}"
-EndProject
-Project("{FAE04EC0-301F-11D3-BF4B-00C04F79EFBC}") = "SharedStreams", "src\DataStreamSuite\SharedStreams\SharedStreams.csproj", "{A8B861DE-E098-4291-98EE-76AA0F5F5A7B}"
-EndProject
-Project("{FAE04EC0-301F-11D3-BF4B-00C04F79EFBC}") = "PubSub", "src\DataStreamSuite\PubSub\PubSub.csproj", "{EFA9F78D-083E-457A-BABB-9386C88C32C0}"
-EndProject
 Global
 	GlobalSection(SolutionConfigurationPlatforms) = preSolution
 		Debug|x64 = Debug|x64
@@ -80,6 +100,102 @@ Global
 		{89D666E6-B500-498F-B026-4D167F68ADFC}.Release|x64.Build.0 = Release|x64
 		{89D666E6-B500-498F-B026-4D167F68ADFC}.RelWithDebInfo|x64.ActiveCfg = Debug|x64
 		{89D666E6-B500-498F-B026-4D167F68ADFC}.RelWithDebInfo|x64.Build.0 = Debug|x64
+		{EF26A475-6101-4C9C-A717-D60BB92A2302}.Debug|x64.ActiveCfg = Debug|Any CPU
+		{EF26A475-6101-4C9C-A717-D60BB92A2302}.Debug|x64.Build.0 = Debug|Any CPU
+		{EF26A475-6101-4C9C-A717-D60BB92A2302}.MinSizeRel|x64.ActiveCfg = Debug|Any CPU
+		{EF26A475-6101-4C9C-A717-D60BB92A2302}.MinSizeRel|x64.Build.0 = Debug|Any CPU
+		{EF26A475-6101-4C9C-A717-D60BB92A2302}.Release|x64.ActiveCfg = Release|Any CPU
+		{EF26A475-6101-4C9C-A717-D60BB92A2302}.Release|x64.Build.0 = Release|Any CPU
+		{EF26A475-6101-4C9C-A717-D60BB92A2302}.RelWithDebInfo|x64.ActiveCfg = Debug|Any CPU
+		{EF26A475-6101-4C9C-A717-D60BB92A2302}.RelWithDebInfo|x64.Build.0 = Debug|Any CPU
+		{911B08A3-515A-441D-84C8-8A43740B5FC1}.Debug|x64.ActiveCfg = Debug|Any CPU
+		{911B08A3-515A-441D-84C8-8A43740B5FC1}.Debug|x64.Build.0 = Debug|Any CPU
+		{911B08A3-515A-441D-84C8-8A43740B5FC1}.MinSizeRel|x64.ActiveCfg = Debug|Any CPU
+		{911B08A3-515A-441D-84C8-8A43740B5FC1}.MinSizeRel|x64.Build.0 = Debug|Any CPU
+		{911B08A3-515A-441D-84C8-8A43740B5FC1}.Release|x64.ActiveCfg = Release|Any CPU
+		{911B08A3-515A-441D-84C8-8A43740B5FC1}.Release|x64.Build.0 = Release|Any CPU
+		{911B08A3-515A-441D-84C8-8A43740B5FC1}.RelWithDebInfo|x64.ActiveCfg = Debug|Any CPU
+		{911B08A3-515A-441D-84C8-8A43740B5FC1}.RelWithDebInfo|x64.Build.0 = Debug|Any CPU
+		{14140310-2233-430A-821B-F3058A25234B}.Debug|x64.ActiveCfg = Debug|Any CPU
+		{14140310-2233-430A-821B-F3058A25234B}.Debug|x64.Build.0 = Debug|Any CPU
+		{14140310-2233-430A-821B-F3058A25234B}.MinSizeRel|x64.ActiveCfg = Debug|Any CPU
+		{14140310-2233-430A-821B-F3058A25234B}.MinSizeRel|x64.Build.0 = Debug|Any CPU
+		{14140310-2233-430A-821B-F3058A25234B}.Release|x64.ActiveCfg = Release|Any CPU
+		{14140310-2233-430A-821B-F3058A25234B}.Release|x64.Build.0 = Release|Any CPU
+		{14140310-2233-430A-821B-F3058A25234B}.RelWithDebInfo|x64.ActiveCfg = Debug|Any CPU
+		{14140310-2233-430A-821B-F3058A25234B}.RelWithDebInfo|x64.Build.0 = Debug|Any CPU
+		{43146EA8-6FCC-46B8-88B0-42F9BB5E6E0A}.Debug|x64.ActiveCfg = Debug|Any CPU
+		{43146EA8-6FCC-46B8-88B0-42F9BB5E6E0A}.Debug|x64.Build.0 = Debug|Any CPU
+		{43146EA8-6FCC-46B8-88B0-42F9BB5E6E0A}.MinSizeRel|x64.ActiveCfg = Debug|Any CPU
+		{43146EA8-6FCC-46B8-88B0-42F9BB5E6E0A}.MinSizeRel|x64.Build.0 = Debug|Any CPU
+		{43146EA8-6FCC-46B8-88B0-42F9BB5E6E0A}.Release|x64.ActiveCfg = Release|Any CPU
+		{43146EA8-6FCC-46B8-88B0-42F9BB5E6E0A}.Release|x64.Build.0 = Release|Any CPU
+		{43146EA8-6FCC-46B8-88B0-42F9BB5E6E0A}.RelWithDebInfo|x64.ActiveCfg = Debug|Any CPU
+		{43146EA8-6FCC-46B8-88B0-42F9BB5E6E0A}.RelWithDebInfo|x64.Build.0 = Debug|Any CPU
+		{6F22DB96-6EAB-4034-99D9-287021C36F01}.Debug|x64.ActiveCfg = Debug|Any CPU
+		{6F22DB96-6EAB-4034-99D9-287021C36F01}.Debug|x64.Build.0 = Debug|Any CPU
+		{6F22DB96-6EAB-4034-99D9-287021C36F01}.MinSizeRel|x64.ActiveCfg = Debug|Any CPU
+		{6F22DB96-6EAB-4034-99D9-287021C36F01}.MinSizeRel|x64.Build.0 = Debug|Any CPU
+		{6F22DB96-6EAB-4034-99D9-287021C36F01}.Release|x64.ActiveCfg = Release|Any CPU
+		{6F22DB96-6EAB-4034-99D9-287021C36F01}.Release|x64.Build.0 = Release|Any CPU
+		{6F22DB96-6EAB-4034-99D9-287021C36F01}.RelWithDebInfo|x64.ActiveCfg = Debug|Any CPU
+		{6F22DB96-6EAB-4034-99D9-287021C36F01}.RelWithDebInfo|x64.Build.0 = Debug|Any CPU
+		{A5C33956-A41F-49B3-A41D-1082CBA59D02}.Debug|x64.ActiveCfg = Debug|Any CPU
+		{A5C33956-A41F-49B3-A41D-1082CBA59D02}.Debug|x64.Build.0 = Debug|Any CPU
+		{A5C33956-A41F-49B3-A41D-1082CBA59D02}.MinSizeRel|x64.ActiveCfg = Debug|Any CPU
+		{A5C33956-A41F-49B3-A41D-1082CBA59D02}.MinSizeRel|x64.Build.0 = Debug|Any CPU
+		{A5C33956-A41F-49B3-A41D-1082CBA59D02}.Release|x64.ActiveCfg = Release|Any CPU
+		{A5C33956-A41F-49B3-A41D-1082CBA59D02}.Release|x64.Build.0 = Release|Any CPU
+		{A5C33956-A41F-49B3-A41D-1082CBA59D02}.RelWithDebInfo|x64.ActiveCfg = Debug|Any CPU
+		{A5C33956-A41F-49B3-A41D-1082CBA59D02}.RelWithDebInfo|x64.Build.0 = Debug|Any CPU
+		{2FE745C3-1EDF-4421-8CD7-A63EE0C1AE43}.Debug|x64.ActiveCfg = Debug|Any CPU
+		{2FE745C3-1EDF-4421-8CD7-A63EE0C1AE43}.Debug|x64.Build.0 = Debug|Any CPU
+		{2FE745C3-1EDF-4421-8CD7-A63EE0C1AE43}.MinSizeRel|x64.ActiveCfg = Debug|Any CPU
+		{2FE745C3-1EDF-4421-8CD7-A63EE0C1AE43}.MinSizeRel|x64.Build.0 = Debug|Any CPU
+		{2FE745C3-1EDF-4421-8CD7-A63EE0C1AE43}.Release|x64.ActiveCfg = Release|Any CPU
+		{2FE745C3-1EDF-4421-8CD7-A63EE0C1AE43}.Release|x64.Build.0 = Release|Any CPU
+		{2FE745C3-1EDF-4421-8CD7-A63EE0C1AE43}.RelWithDebInfo|x64.ActiveCfg = Debug|Any CPU
+		{2FE745C3-1EDF-4421-8CD7-A63EE0C1AE43}.RelWithDebInfo|x64.Build.0 = Debug|Any CPU
+		{FDA791E0-2B91-4AD5-B91D-CF920A4C4E38}.Debug|x64.ActiveCfg = Debug|Any CPU
+		{FDA791E0-2B91-4AD5-B91D-CF920A4C4E38}.Debug|x64.Build.0 = Debug|Any CPU
+		{FDA791E0-2B91-4AD5-B91D-CF920A4C4E38}.MinSizeRel|x64.ActiveCfg = Debug|Any CPU
+		{FDA791E0-2B91-4AD5-B91D-CF920A4C4E38}.MinSizeRel|x64.Build.0 = Debug|Any CPU
+		{FDA791E0-2B91-4AD5-B91D-CF920A4C4E38}.Release|x64.ActiveCfg = Release|Any CPU
+		{FDA791E0-2B91-4AD5-B91D-CF920A4C4E38}.Release|x64.Build.0 = Release|Any CPU
+		{FDA791E0-2B91-4AD5-B91D-CF920A4C4E38}.RelWithDebInfo|x64.ActiveCfg = Debug|Any CPU
+		{FDA791E0-2B91-4AD5-B91D-CF920A4C4E38}.RelWithDebInfo|x64.Build.0 = Debug|Any CPU
+		{B945A183-2174-4ACB-B289-8A5C546656FE}.Debug|x64.ActiveCfg = Debug|Any CPU
+		{B945A183-2174-4ACB-B289-8A5C546656FE}.Debug|x64.Build.0 = Debug|Any CPU
+		{B945A183-2174-4ACB-B289-8A5C546656FE}.MinSizeRel|x64.ActiveCfg = Debug|Any CPU
+		{B945A183-2174-4ACB-B289-8A5C546656FE}.MinSizeRel|x64.Build.0 = Debug|Any CPU
+		{B945A183-2174-4ACB-B289-8A5C546656FE}.Release|x64.ActiveCfg = Release|Any CPU
+		{B945A183-2174-4ACB-B289-8A5C546656FE}.Release|x64.Build.0 = Release|Any CPU
+		{B945A183-2174-4ACB-B289-8A5C546656FE}.RelWithDebInfo|x64.ActiveCfg = Debug|Any CPU
+		{B945A183-2174-4ACB-B289-8A5C546656FE}.RelWithDebInfo|x64.Build.0 = Debug|Any CPU
+		{39AEA075-6457-46C0-8821-1158BE6EFA57}.Debug|x64.ActiveCfg = Debug|Any CPU
+		{39AEA075-6457-46C0-8821-1158BE6EFA57}.Debug|x64.Build.0 = Debug|Any CPU
+		{39AEA075-6457-46C0-8821-1158BE6EFA57}.MinSizeRel|x64.ActiveCfg = Debug|Any CPU
+		{39AEA075-6457-46C0-8821-1158BE6EFA57}.MinSizeRel|x64.Build.0 = Debug|Any CPU
+		{39AEA075-6457-46C0-8821-1158BE6EFA57}.Release|x64.ActiveCfg = Release|Any CPU
+		{39AEA075-6457-46C0-8821-1158BE6EFA57}.Release|x64.Build.0 = Release|Any CPU
+		{39AEA075-6457-46C0-8821-1158BE6EFA57}.RelWithDebInfo|x64.ActiveCfg = Debug|Any CPU
+		{39AEA075-6457-46C0-8821-1158BE6EFA57}.RelWithDebInfo|x64.Build.0 = Debug|Any CPU
+		{9F6D7DE8-185B-4C1D-B809-3314D092DF28}.Debug|x64.ActiveCfg = Debug|Any CPU
+		{9F6D7DE8-185B-4C1D-B809-3314D092DF28}.Debug|x64.Build.0 = Debug|Any CPU
+		{9F6D7DE8-185B-4C1D-B809-3314D092DF28}.MinSizeRel|x64.ActiveCfg = Debug|Any CPU
+		{9F6D7DE8-185B-4C1D-B809-3314D092DF28}.MinSizeRel|x64.Build.0 = Debug|Any CPU
+		{9F6D7DE8-185B-4C1D-B809-3314D092DF28}.Release|x64.ActiveCfg = Release|Any CPU
+		{9F6D7DE8-185B-4C1D-B809-3314D092DF28}.Release|x64.Build.0 = Release|Any CPU
+		{9F6D7DE8-185B-4C1D-B809-3314D092DF28}.RelWithDebInfo|x64.ActiveCfg = Debug|Any CPU
+		{9F6D7DE8-185B-4C1D-B809-3314D092DF28}.RelWithDebInfo|x64.Build.0 = Debug|Any CPU
+		{6BD3F6D8-B6FC-44E8-B8FE-06352F638B9B}.Debug|x64.ActiveCfg = Debug|Any CPU
+		{6BD3F6D8-B6FC-44E8-B8FE-06352F638B9B}.Debug|x64.Build.0 = Debug|Any CPU
+		{6BD3F6D8-B6FC-44E8-B8FE-06352F638B9B}.MinSizeRel|x64.ActiveCfg = Debug|Any CPU
+		{6BD3F6D8-B6FC-44E8-B8FE-06352F638B9B}.MinSizeRel|x64.Build.0 = Debug|Any CPU
+		{6BD3F6D8-B6FC-44E8-B8FE-06352F638B9B}.Release|x64.ActiveCfg = Release|Any CPU
+		{6BD3F6D8-B6FC-44E8-B8FE-06352F638B9B}.Release|x64.Build.0 = Release|Any CPU
+		{6BD3F6D8-B6FC-44E8-B8FE-06352F638B9B}.RelWithDebInfo|x64.ActiveCfg = Debug|Any CPU
+		{6BD3F6D8-B6FC-44E8-B8FE-06352F638B9B}.RelWithDebInfo|x64.Build.0 = Debug|Any CPU
 		{CCDCB00A-CABF-4D82-8A5F-33EEC8ACE51D}.Debug|x64.ActiveCfg = Debug|Any CPU
 		{CCDCB00A-CABF-4D82-8A5F-33EEC8ACE51D}.Debug|x64.Build.0 = Debug|Any CPU
 		{CCDCB00A-CABF-4D82-8A5F-33EEC8ACE51D}.MinSizeRel|x64.ActiveCfg = Debug|Any CPU
@@ -88,38 +204,6 @@ Global
 		{CCDCB00A-CABF-4D82-8A5F-33EEC8ACE51D}.Release|x64.Build.0 = Release|Any CPU
 		{CCDCB00A-CABF-4D82-8A5F-33EEC8ACE51D}.RelWithDebInfo|x64.ActiveCfg = Debug|Any CPU
 		{CCDCB00A-CABF-4D82-8A5F-33EEC8ACE51D}.RelWithDebInfo|x64.Build.0 = Debug|Any CPU
-		{CB62D5D2-3495-4488-BFBC-7C7A3725A1EE}.Debug|x64.ActiveCfg = Debug|Any CPU
-		{CB62D5D2-3495-4488-BFBC-7C7A3725A1EE}.Debug|x64.Build.0 = Debug|Any CPU
-		{CB62D5D2-3495-4488-BFBC-7C7A3725A1EE}.MinSizeRel|x64.ActiveCfg = Debug|Any CPU
-		{CB62D5D2-3495-4488-BFBC-7C7A3725A1EE}.MinSizeRel|x64.Build.0 = Debug|Any CPU
-		{CB62D5D2-3495-4488-BFBC-7C7A3725A1EE}.Release|x64.ActiveCfg = Release|Any CPU
-		{CB62D5D2-3495-4488-BFBC-7C7A3725A1EE}.Release|x64.Build.0 = Release|Any CPU
-		{CB62D5D2-3495-4488-BFBC-7C7A3725A1EE}.RelWithDebInfo|x64.ActiveCfg = Debug|Any CPU
-		{CB62D5D2-3495-4488-BFBC-7C7A3725A1EE}.RelWithDebInfo|x64.Build.0 = Debug|Any CPU
-		{05995ED5-650A-4955-9DFC-3242B737E77A}.Debug|x64.ActiveCfg = Debug|Any CPU
-		{05995ED5-650A-4955-9DFC-3242B737E77A}.Debug|x64.Build.0 = Debug|Any CPU
-		{05995ED5-650A-4955-9DFC-3242B737E77A}.MinSizeRel|x64.ActiveCfg = Debug|Any CPU
-		{05995ED5-650A-4955-9DFC-3242B737E77A}.MinSizeRel|x64.Build.0 = Debug|Any CPU
-		{05995ED5-650A-4955-9DFC-3242B737E77A}.Release|x64.ActiveCfg = Release|Any CPU
-		{05995ED5-650A-4955-9DFC-3242B737E77A}.Release|x64.Build.0 = Release|Any CPU
-		{05995ED5-650A-4955-9DFC-3242B737E77A}.RelWithDebInfo|x64.ActiveCfg = Debug|Any CPU
-		{05995ED5-650A-4955-9DFC-3242B737E77A}.RelWithDebInfo|x64.Build.0 = Debug|Any CPU
-		{A8B861DE-E098-4291-98EE-76AA0F5F5A7B}.Debug|x64.ActiveCfg = Debug|Any CPU
-		{A8B861DE-E098-4291-98EE-76AA0F5F5A7B}.Debug|x64.Build.0 = Debug|Any CPU
-		{A8B861DE-E098-4291-98EE-76AA0F5F5A7B}.MinSizeRel|x64.ActiveCfg = Debug|Any CPU
-		{A8B861DE-E098-4291-98EE-76AA0F5F5A7B}.MinSizeRel|x64.Build.0 = Debug|Any CPU
-		{A8B861DE-E098-4291-98EE-76AA0F5F5A7B}.Release|x64.ActiveCfg = Release|Any CPU
-		{A8B861DE-E098-4291-98EE-76AA0F5F5A7B}.Release|x64.Build.0 = Release|Any CPU
-		{A8B861DE-E098-4291-98EE-76AA0F5F5A7B}.RelWithDebInfo|x64.ActiveCfg = Debug|Any CPU
-		{A8B861DE-E098-4291-98EE-76AA0F5F5A7B}.RelWithDebInfo|x64.Build.0 = Debug|Any CPU
-		{EFA9F78D-083E-457A-BABB-9386C88C32C0}.Debug|x64.ActiveCfg = Debug|Any CPU
-		{EFA9F78D-083E-457A-BABB-9386C88C32C0}.Debug|x64.Build.0 = Debug|Any CPU
-		{EFA9F78D-083E-457A-BABB-9386C88C32C0}.MinSizeRel|x64.ActiveCfg = Debug|Any CPU
-		{EFA9F78D-083E-457A-BABB-9386C88C32C0}.MinSizeRel|x64.Build.0 = Debug|Any CPU
-		{EFA9F78D-083E-457A-BABB-9386C88C32C0}.Release|x64.ActiveCfg = Release|Any CPU
-		{EFA9F78D-083E-457A-BABB-9386C88C32C0}.Release|x64.Build.0 = Release|Any CPU
-		{EFA9F78D-083E-457A-BABB-9386C88C32C0}.RelWithDebInfo|x64.ActiveCfg = Debug|Any CPU
-		{EFA9F78D-083E-457A-BABB-9386C88C32C0}.RelWithDebInfo|x64.Build.0 = Debug|Any CPU
 	EndGlobalSection
 	GlobalSection(SolutionProperties) = preSolution
 		HideSolutionNode = FALSE
@@ -130,4 +214,19 @@ Global
 	GlobalSection(Performance) = preSolution
 		HasPerformanceSessions = true
 	EndGlobalSection
+	GlobalSection(NestedProjects) = preSolution
+		{C09ED521-D2DF-40BC-A76F-EB3CB870B2F6} = {267F81EF-A8AB-49FD-9F56-D7EC3C1F7C7C}
+		{EF26A475-6101-4C9C-A717-D60BB92A2302} = {C09ED521-D2DF-40BC-A76F-EB3CB870B2F6}
+		{911B08A3-515A-441D-84C8-8A43740B5FC1} = {C09ED521-D2DF-40BC-A76F-EB3CB870B2F6}
+		{14140310-2233-430A-821B-F3058A25234B} = {C09ED521-D2DF-40BC-A76F-EB3CB870B2F6}
+		{43146EA8-6FCC-46B8-88B0-42F9BB5E6E0A} = {C09ED521-D2DF-40BC-A76F-EB3CB870B2F6}
+		{6F22DB96-6EAB-4034-99D9-287021C36F01} = {C09ED521-D2DF-40BC-A76F-EB3CB870B2F6}
+		{A5C33956-A41F-49B3-A41D-1082CBA59D02} = {C09ED521-D2DF-40BC-A76F-EB3CB870B2F6}
+		{2FE745C3-1EDF-4421-8CD7-A63EE0C1AE43} = {C09ED521-D2DF-40BC-A76F-EB3CB870B2F6}
+		{FDA791E0-2B91-4AD5-B91D-CF920A4C4E38} = {C09ED521-D2DF-40BC-A76F-EB3CB870B2F6}
+		{B945A183-2174-4ACB-B289-8A5C546656FE} = {C09ED521-D2DF-40BC-A76F-EB3CB870B2F6}
+		{39AEA075-6457-46C0-8821-1158BE6EFA57} = {C09ED521-D2DF-40BC-A76F-EB3CB870B2F6}
+		{9F6D7DE8-185B-4C1D-B809-3314D092DF28} = {C09ED521-D2DF-40BC-A76F-EB3CB870B2F6}
+		{6BD3F6D8-B6FC-44E8-B8FE-06352F638B9B} = {C09ED521-D2DF-40BC-A76F-EB3CB870B2F6}
+	EndGlobalSection
 EndGlobal
@@ -112,7 +112,7 @@ function(init_library)
  )

  # Add tests executable (if there are any)
-  if (${_name}_TEST_SOURCES)
+  if (${_name}_TEST_SOURCES AND NOT WIN32)
    add_executable(${_name}-tests ${${_name}_TEST_SOURCES})

    get_target_property(_lib_type ${_name} TYPE)
@@ -15,7 +15,11 @@ function(protobuf_generate_cpp)
  cmake_parse_arguments(_ARGS "${_options}" "${_one_value_args}" "${_multi_value_args}" ${ARGN})

  if (NOT DEFINED _ARGS_PROTOC)
-    set(_ARGS_PROTOC "${EXTERNALS_BIN}/protoc")
+    if (WIN32)
+      set(_ARGS_PROTOC "${EXTERNALS_BIN}/protoc.exe")
+    else()
+      set(_ARGS_PROTOC "${EXTERNALS_BIN}/protoc")
+    endif()
  endif()
  set(${_ARGS_CXX_SOURCES})
  set(${_ARGS_CXX_HEADERS})
@@ -5,7 +5,7 @@ cmake_minimum_required(VERSION 3.5)
 cmake_policy(SET CMP0076 NEW)

 # add protos to the library to facilitate source_group
-target_sources(libspark PUBLIC ${Config_Protos} ${NonConfig_Protos} ${Service_Protos})
+target_sources(spark PUBLIC ${Config_Protos} ${NonConfig_Protos} ${Service_Protos})

 function(assign_source_group)
  foreach(_source IN ITEMS ${ARGN})
@@ -3,52 +3,55 @@ services:
    build:
      dockerfile: src/app_sparkd/Dockerfile
      args:
-        - SOURCE_DIR=install/sparkd-release-gcc
-        - CONF_FILE=src/app_sparkd/spark.conf
+        - IMAGE=${SPARKD__IMAGE}
+        - SOURCE_DIR=${SPARKD__SOURCE_DIR}
+        - CONF_FILE=${SPARKD__CONF_FILE}
+        - MD_RECORDING_DIR=${SPARKD__MD_RECORDING_DIR}
    command: [ "./sparkd", "run" ]
    ports:
-      - 8000:8000
-      - 8001:8001
-      - 8002:8002
+      - "8000:8000"
+      - "8001:8001"
+      - "8002:8002"

  pubsub:
    environment:
-      - DOTNET_ENVIRONMENT=Staging
+      - DOTNET_ENVIRONMENT=${PUBSUB__DOTNET_ENVIRONMENT}
    build:
      dockerfile: src/DataStreamSuite/PubSubServer/Dockerfile
      args:
-        - SOURCE_DIR=build/Release/bin/net7.0/linux-x64/publish
-        - CONF_FILE=src/DataStreamSuite/PubSubServer/spark.conf
+        - CS_SOURCE_DIR=${PUBSUB__CS_SOURCE_DIR}
+        - CPP_SOURCE_DIR=${PUBSUB__CPP_SOURCE_DIR}
+        - CONF_FILE=${PUBSUB__CONF_FILE}
    ports:
-      - 22019:22019
+      - "22019:22019"
    command: [ "./PubSubServer" ]

  md-publisher:
    build:
      dockerfile: src/DataStreamSuite/DataStreamPublisher/Dockerfile
      args:
-        - SOURCE_DIR=build/Release/bin/net7.0/linux-x64/publish
-        - CONF_FILE=src/DataStreamSuite/DataStreamPublisher/spark.conf
+        - SOURCE_DIR=${MD_PUBLISHER__SOURCE_DIR}
+        - CONF_FILE=${MD_PUBLISHER__CONF_FILE}
    depends_on:
      - pubsub
      - sparkd
    command: [ "dockerize", "-wait", "tcp://pubsub:22019", "-wait", "tcp://sparkd:8000", "-timeout", "10s", "./DataStreamPublisher" ]
    environment:
-      - DOTNET_ENVIRONMENT=MarketData
+      - DOTNET_ENVIRONMENT=${MD_PUBLISHER__DOTNET_ENVIRONMENT}

  webapp:
    ports:
-      - 8080:8080
-      - 8443:8443
+      - "8080:8080"
+      - "8443:8443"
    build:
      dockerfile: src/DataStreamSuite/Bts.DataStreamWebApp.Server/Dockerfile
      args:
-        - SOURCE_DIR=build/Release/bin/net7.0/linux-x64/publish
-        - WEB_ROOT=build/Bts.DataStreamWebApp.Client/production/dist
-        - CONF_FILE=src/DataStreamSuite/Bts.DataStreamWebApp.Server/spark.conf
+        - SOURCE_DIR=${WEBAPP__SOURCE_DIR}
+        - WEB_ROOT=${WEBAPP__WEB_ROOT}
+        - CONF_FILE=${WEBAPP__CONF_FILE}
    command: [ "./Bts.DataStreamWebApp.Server" ]
    environment:
-      - ASPNETCORE_ENVIRONMENT=Staging
+      - ASPNETCORE_ENVIRONMENT=${WEBAPP__ASPNETCORE_ENVIRONMENT}

  api-server:
    depends_on:
@@ -60,7 +63,7 @@ services:
      dockerfile: src/client/spark_api_server/Dockerfile
    command: [ "dockerize", "-wait", "tcp://sparkd:8000", "-timeout", "10s", "./spark_api_server", "run" ]
    environment:
-      - ASPNETCORE_ENVIRONMENT=Staging
+      - ASPNETCORE_ENVIRONMENT=${API_SERVER__ASPNETCORE_ENVIRONMENT}

  sparkd-ping-test:
    depends_on:
@@ -1,7 +1,7 @@
 import PropTypes from 'prop-types';
 import { useTheme } from '@mui/material/styles';
 import { DataGrid } from '@mui/x-data-grid';
-import './DataGrid.css'
+import './DataGrid.css';
 import { useMemo, useState } from 'react';

 export default function DataTable({ columns, rows }) {
@@ -23,7 +23,7 @@ export default function DataTable({ columns, rows }) {
        headerName: col.name,
        width: columnWidths[col.name],
      };
-    }
+    };
  }, [columnWidths]);

  return (
@@ -12,7 +12,7 @@ const QueryStatusView = ({ queryState, details, error }) => {

  let header = queryState;
  let headerContent = null;
-  let detailElements = [];
+  const detailElements = [];
  let severity = queryState == "Subscribed" ? "success" : "info";
  let cursor = details == null ? "default" : "pointer";
  if (error != null) {
@@ -20,14 +20,14 @@ const QueryStatusView = ({ queryState, details, error }) => {
    cursor = "pointer";
    header = `Error Code: ${error.code}`;
    headerContent = error.message;
-    detailElements.push(<Typography variant="body1">{error.data.error_message}</Typography>);
-    detailElements.push(<Typography variant="subtitle1">Stack Trace:</Typography>);
-    (error.data.stack_trace ?? []).forEach(line => detailElements.push(
-      <Typography variant="body2" component="pre" sx={{ whiteSpace: 'pre-wrap' }}>
+    detailElements.push(<Typography key="error-msg" variant="body1">{error.data.error_message}</Typography>);
+    detailElements.push(<Typography key="error-details" variant="subtitle1">Stack Trace:</Typography>);
+    (error.data.stack_trace ?? []).forEach((line, idx) => detailElements.push(
+      <Typography key={`stacktrace-${idx}`} variant="body2" component="pre" sx={{ whiteSpace: 'pre-wrap' }}>
        {line}
      </Typography>));
  } else if (details != null) {
-    detailElements.push(<Typography variant="body1">{details}</Typography>);
+    detailElements.push(<Typography key="status-msg" variant="body1">{details}</Typography>);
  }
  return (
    <Box sx={{ margin: theme.spacing(2, 0, 0, 0) }}>
@@ -13,7 +13,7 @@ function getGridColDef(col) {
 export default function ReactDataGrid({columns, rows}) {
  return (
    <DataGrid className={"data-grid"} rows={rows} columns={columns.map(getGridColDef)} />
-  )
+  );
 }

 ReactDataGrid.propTypes = {
@@ -3,7 +3,7 @@ import Button from '@mui/material/Button';
 import { useTheme } from '@mui/material/styles';
 import PropTypes from 'prop-types';
 import Tooltip from '@mui/material/Tooltip';
-import ChevronLeftRounded from "@mui/icons-material/ChevronLeftRounded"
+import ChevronLeftRounded from "@mui/icons-material/ChevronLeftRounded";
 import ChevronRightRounded from '@mui/icons-material/ChevronRightRounded';
 import FileCopyIcon from '@mui/icons-material/FileCopy';
 import './SideBar.css';
@@ -51,7 +51,7 @@ const Sidebar = ({ onSelection, onOpened, onClosed }) => {
      } catch (error) {
        console.error('Error fetching WebSocket config:', error);
      }
-    }
+    };
    fetchCannedQueries();
  }, []);

@@ -51,11 +51,26 @@ export default defineConfig(({ $$command, mode, $$isSsrBuild, $$isPreview }) =>
      proxy: {
        '^/api': {
          target,
-          secure: false
+          secure: false,
+          configure: (proxy) => {
+            proxy.on('error', (err) => {
+              console.log('proxy error', err);
+            });
+            proxy.on('proxyReq', (proxyReq, req) => {
+              console.log(`Forwarding ${proxyReq.method} ${proxyReq.protocol}//${proxyReq.host}${proxyReq.path} to ${req.socket.remoteAddress}${req.url}`);
+            });
+            proxy.on('proxyRes', (proxyRes, req, res) => {
+              console.log(`Forwarding response ${req.socket.localAddress} to ${res.socket.remoteAddress}. Status=${proxyRes.statusCode}`);
+            });
+          },
        },
        '^/ws': {
          target,
          secure: false
+        },
+        '^/browse': {
+          target,
+          secure: false
        }
      },
      port: 5173,
@@ -1,14 +1,21 @@
 namespace Bts.DataStreamWebApp.Server.Controllers;

+using Bts.DataStream;
+
 using Microsoft.AspNetCore.Mvc;

 [Route("/ws")]
 public class DataStreamSubscriptionController : ControllerBase {
   readonly ILogger logger;
+   readonly IDataStreamFactory streamFactory;

-   public DataStreamSubscriptionController(ILogger<DataStreamSubscriptionController> logger) {
+   public DataStreamSubscriptionController(ILogger<DataStreamSubscriptionController> logger, IEnumerable<IDataStreamFactory> streamFactories) {
      this.logger = logger;
      logger.LogInformation("Creating {ControllerType}", GetType().Name);
+      var streamFactoryList = streamFactories.ToList();
+      streamFactory = streamFactoryList.Count == 1
+         ? streamFactoryList[0]
+         : new CompositeDataStreamFactory(streamFactoryList);
   }

   [Route("/ws")]
@@ -21,7 +28,7 @@ public class DataStreamSubscriptionController : ControllerBase {
      try {
         using var webSocket = await HttpContext.WebSockets.AcceptWebSocketAsync();
         logger.LogInformation("Websocket connection established.");
-         using var server = new DataStreamWebSocketServer(webSocket, logger);
+         using var server = new DataStreamWebSocketServer(webSocket, streamFactory, logger);
         await server.Run(cancellationToken);
      } catch (Exception ex) {
         logger.LogError("DataStreamSubscriptionController.Run threw exception: {Exception}", ex);
@@ -2,6 +2,7 @@ namespace Bts.DataStreamWebApp.Server.Controllers;

 using bts.websocket_core.json_rpc;

+using Bts.DataStream;
 using Bts.DataStreamWebApp.Server.Messages;

 using System.Net.WebSockets;
@@ -13,9 +14,11 @@ public class DataStreamWebSocketServer : IDisposable, IAsyncDisposable {
   readonly ILogger logger;
   readonly WebSocket webSocket;
   readonly Dictionary<string, DataStreamSubscription> subscriptions = new();
+   readonly IDataStreamFactory streamFactory;

-   public DataStreamWebSocketServer(WebSocket webSocket, ILogger logger) {
+   public DataStreamWebSocketServer(WebSocket webSocket, IDataStreamFactory streamFactory, ILogger logger) {
      this.webSocket = webSocket;
+      this.streamFactory = streamFactory;
      this.logger = logger;
   }

@@ -67,7 +70,7 @@ public class DataStreamWebSocketServer : IDisposable, IAsyncDisposable {
      }
      var subId = subReq.SubscriptionId ?? throw new InvalidOperationException("SubscriptionId is null (parse false)");
      disposeSubscription(subId);
-      if (!subReq.ParseQuery(out var vs, out parseErr, out errCode)) {
+      if (!subReq.ParseQuery(streamFactory, out var vs, out parseErr, out errCode)) {
         logger.LogError("Invalid 'subscribe' request: {Reason}", parseErr);
         var rpcError = JsonRpcResponse.CreateError<SubscriptionError>(
            req.Id, new() { Code = (long)errCode, Message = "Failed to parse subscription request", Data = parseErr });
@@ -4,6 +4,7 @@ using BCG;

 using bts.websocket_core.json_rpc;

+using Bts.DataStream;
 using Bts.DataStreamExpressions;

 using System.Diagnostics.CodeAnalysis;
@@ -37,7 +38,12 @@ public class SubscribeRequest {
      return true;
   }

-   internal bool ParseQuery([NotNullWhen(true)] out ViewStream? stream, [NotNullWhen(false)] out SubscriptionError? error, out ErrorCode errorCode) {
+   internal bool ParseQuery(
+      IDataStreamFactory streamFactory,
+      [NotNullWhen(true)] out ViewStream? stream,
+      [NotNullWhen(false)] out SubscriptionError? error,
+      out ErrorCode errorCode
+   ) {
      stream = null;
      errorCode = ErrorCode.None;
      error = null;
@@ -54,7 +60,7 @@ public class SubscribeRequest {
      }
      Exception? exception = null;
      try {
-         stream = exp.getView(new Dictionary<string, StatementExpression>(0));
+         stream = exp.getView(new Dictionary<string, StatementExpression>(0), streamFactory);
      } catch (Exception ex) {
         exception = ex;
      }
@@ -1,5 +1,6 @@
 using Bts.DataStreamCore.Services;
 using Bts.PubSub.Services;
+using Bts.SharedStreams.Services;

 using Spark.Api.Services;
 using Spark.Core.Services;
@@ -78,5 +79,9 @@ static void ConfigureServices(IServiceCollection services, IConfiguration config
      .AddSparkApi(config.GetSection("SparkApi"))
      .AddMasterWorker(config.GetSection("MasterWorker"))
      .AddViewStreamDatabase()
+      .AddPrimitiveDataStreamFactory(config.GetSection("PrimitiveStreams"))
+      .AddOhlcvStreams(config.GetSection("StreamDatabase"))
+      .AddNaturalLanguageStreams(config.GetSection("NaturalLanguageStream"))
+      .AddCompositeDataStreamFactory()
      .AddPubSubClient(config.GetSection("PubSub"));
 }
@@ -0,0 +1,45 @@
+{
+  "NaturalLanguagePrompt": {
+    "prompts": [
+      {
+        "category": "Impact of financial events on the market",
+        "description": "Show how trading metrics (e.g., price movement, volume) are affected by events related to finance (rate announcements, earnings reports, etc.).",
+        "prompt": "Show me the price movement and volume of Crude Oil futures during the 5-minute window following the last 5 OPEC meetings",
+        "examples": [
+          {
+            "natural_language": "Show me the price movement and volume of Crude Oil futures during the 5-minute window following the last 5 OPEC meetings",
+            "dsl": [
+              "time_idx=EventTimeIndex.OpecMeetings(which: \"last 5\", duration: \"+5min\")",
+              "contract_idx=ActiveFutureContracts(predicate: \"product==CL\")",
+              "Bars(time_index: time_idx, contract_index: contract_idx, fields: [\"price_movement\", \"volume\"])"
+            ]
+          },
+          {
+            "natural_language": "How do OPEC meetings affect the price of Crude futures? Examine the 10 minutes after each meeting this year.",
+            "dsl": [
+              "time_idx=EventTimeIndex.OpecMeetings(duration: \"+10min\", predicate: \"timestamp.year==2024\")",
+              "contract_idx=ActiveFutureContracts(predicate: \"product==CL\")",
+              "Bars(time_index: time_idx, contract_index: contract_idx, fields: [\"price_movement\", \"volume\"])"
+            ]
+          },
+          {
+            "natural_language": "Show me how the SP 500 price moves after FOMC rate announcements that day",
+            "dsl": [
+              "time_idx=EventTimeIndex.FomcRateAnnouncements(which: \"timestamp.year==2024\", duration: \"+10min\")",
+              "contract_idx=ActiveFutureContracts(predicate: \"product==ES\")",
+              "Bars(time_index: time_idx, contract_index: contract_idx, fields: [\"price_movement\", \"volume\"])"
+            ]
+          },
+          {
+            "natural_language": "Analyze the impact of the FED's interest rate hikes in 2022 on the 10-year Treasury yield, focusing on the 2-week period following each hike.",
+            "dsl": [
+              "time_idx=FedInterestRateAnnouncements(predicate: \"timestamp.year==2022 && change_pct>0\", duration: \"+2 weeks\")",
+              "contract_idx=ActiveFutureContracts(predicate: \"product==ZN\")",
+              "Bars(time_index: time_idx, contract_index: contract_idx)"
+            ]
+          }
+        ]
+      }
+    ]
+  }
+}
@@ -11,8 +11,14 @@
      "Http": {
        "Url": "http://0.0.0.0:8080"
      },
+      "HttpIpv6": {
+        "Url": "http://[::1]:8080"
+      },
      "Https": {
        "Url": "https://0.0.0.0:8443"
+      },
+      "HttpsIpv6": {
+        "Url": "https://[::1]:8443"
      }
    }
  },
@@ -22,6 +28,358 @@
  "ViewStreamDatabase": {
    "Interval": "00:00:00.1"
  },
+  "PrimitiveStream": {
+    "Assemblies": [
+      "DataStream.dll",
+      "SharedStreams.dll"
+    ],
+    "ExcludedTypes": [
+      "SettlementPriceStream",
+      "OHLCVStream"
+    ]
+  },
+  "StreamDatabase": {
+    "Path": "/home/bts/.local/share/bts/silver_pine/store.db",
+    "Query": "select * from metadata"
+  },
+  "NaturalLanguageStream": {
+    "prompts": [
+      {
+        "category": "Settlement",
+        "MulitLineDescription": [
+          "This is a few shot learning example for an NL to DSL. The DSL function in question is Settlement (symbol_or_contract, datetime1). ",
+          "The first argument is a future (like CL, for crude oil) or a specific future contract like crude oil Dec 24 future (CLZ4). ",
+          "The second argument is the date at which the settlement is to be fetched. There is an overloaded function, Settlement ",
+          "(symbol_or_contract, begin_dt, end_dt) which gets the settlements for all dates from begin_dt to end_dt. This will also have ",
+          "datetimes expressed as Today, Yesterday, Last Friday, First Friday of the month, Last Thursday of August etc. You will need to know ",
+          "what Today is, since there will be referecnes to last week, yesterday etc. These examples are assuming Today is 10/11/2024, however ",
+          "you should use the Today's real date"
+        ],
+        "prompt": "What was the settlement for SP 500 Future on June 8th?",
+        "examples": [
+          {
+            "natural_language": "Find Settlement price of the SP500 future on 04/22/2023.",
+            "dsl": "Settlement(\"ES\", 4/22/2023)"
+          },
+          {
+            "natural_language": "Find Settlement price of the Silver Dec 24 Future on 04/22/2023.",
+            "dsl": "Settlement(\"SIZ4\", 4/22/2023)"
+          },
+          {
+            "natural_language": "Find Yesterday's Settlement of the Crude Oil Future.",
+            "dsl": "Settlement(\"CL\", 10/10/2024)"
+          },
+          {
+            "natural_language": "Find today's settlement of the 10 year treasury future.",
+            "dsl": "Settlement(\"ZN\", 10/11/2024)"
+          },
+          {
+            "natural_language": "Find the Settlement of Natural Gas Dec 24 future on the first Friday of Sept.",
+            "dsl": "Settlement(\"NGZ4\", 9/6/2024)"
+          },
+          {
+            "natural_language": "Find the Settlement of WTI Jan 25 future in the first Week of Sept.",
+            "dsl": "Settlement(\"CLF5\", 9/2/2024, 9/6/2024)"
+          },
+          {
+            "natural_language": "Find the Settlement price for the Sp 500 Dec 24 Mar 25  spread for the first two weeks of September.",
+            "dsl": "Settlement(\"ESZ4-ESH5\", 9/14/2024, 9/1/2024)"
+          },
+          {
+            "natural_language": "Find the Settlement of Natural Gas future on the first Friday of Sept.",
+            "dsl": "Settlement(\"NG\", 9/6/2024)"
+          }
+        ]
+      },
+      {
+        "category": "BBO",
+        "MulitLineDescription": [
+          "This is a few shot learning example for an NL to DSL. The DSL function in question is Bbo(args...). ",
+          "It accepts the following parameters: ",
+          "symbol (a product code such as ZN for 10 year T-Note futures), ",
+          "contract (a specific contract, such as ZNZ4 for the 10 Year T-Note future Dec 2024 exp), ",
+          "venue (name of the exchange, such as CME, NASDAQ, NYSE, NYMEX, DERIBIT etc.), ",
+          "interval (sample frequency of the data), ",
+          "begin and end (inclusive beginning and end of the time window to query, respectively), ",
+          "Arguments are specified by name, for example: Bbo(symbol: \"ZN\", venue: \"CME\", interval: \"PT1M\", begin: \"2024-10-10\", end: \"2024-10-11\"). ",
+          "Depending on the wording of the request, some arguments may be omitted. ",
+          "The begin, end, and interval args should be expressed in iso 8601 format. ",
+          "Interpret dates according to the trading schedule rather than calendar date whenever possible. ",
+          "In some prompts, the time window will be expressed in relative terms (e.g., \"last Monday\", or \"yesterday\") ",
+          "which must be translated to exact dates. To correctly interpret the dsl in the examples, assume the current time is 2024-10-11T12:00-05:00."
+        ],
+        "prompt": "Show me bbo for SP 500 futures on Sep 23rd of this year.",
+        "examples": [
+          {
+            "natural_language": "Get SP500 future bbo from Wednesday.",
+            "dsl": "Bbo(symbol: \"ES\", venue: \"CME\", begin: \"2024-10-08T17:00:00-06:00\", end: \"2024-10-09T16:00:00-06:00\")"
+          },
+          {
+            "natural_language": "Get bbo for 10 year T-Notes in the first minute after the market opened yesterday.",
+            "dsl": "Bbo(symbol: \"ES\", venue: \"CME\", begin: \"2024-10-10T17:00:00-06:00\", end: \"2024-10-10T17:01:00-06:00\")"
+          },
+          {
+            "natural_language": "Retrieve NVIDIA L1 books from Oct 3",
+            "dsl": "Bbo(symbol: \"NVDA\", venue: \"NASDAQ\", begin: \"2024-10-02TT18:00:00-05:00\", end: \"2024-10-03T17:00:00-05:00\")"
+          },
+          {
+            "natural_language": "Show me L1 data for Crude oil futures this week",
+            "dsl": "Bbo(symbol: \"CL\", venue: \"CME\", begin: \"2024-10-06:T17:00:00-06:00\")"
+          },
+          {
+            "natural_language": "Fetch hourly bbo for Gold futures this week",
+            "dsl": "Bbo(symbol: \"GC\", venue: \"CME\", begin: \"2024-10-06:T17:00:00-06:00\", interval: \"PT1H\")"
+          },
+          {
+            "natural_language": "What was the minute by minute bbo for Tesla yesterday?",
+            "dsl": "Bbo(symbol: \"TSLA\", venue: \"NASDAQ\", begin: \"2024-10-09T18:00:00-05:00\", end: \"2024-10-10T17:00:00-05:00\", interval: \"PT1M\")"
+          }
+        ]
+      },
+      {
+        "category": "Order flow imbalance",
+        "MulitLineDescription": [
+          "This is a few shot learning example for an NL to DSL. The DSL function in question is OrderImbalance(args...). ",
+          "It accepts the following parameters: ",
+          "symbol (a product code such as ZN for 10 year T-Note futures), ",
+          "contract (a specific contract, such as ZNZ4 for the 10 Year T-Note future Dec 2024 exp), ",
+          "venue (name of the exchange, such as CME, NASDAQ, NYSE, NYMEX, DERIBIT etc.), ",
+          "interval (sample frequency of the data), ",
+          "levels (the number of price levels to examine)",
+          "begin and end (inclusive beginning and end of the time window to query, respectively), ",
+          "Arguments are specified by name, for example: OrderImbalance(symbol: \"ZN\", venue: \"CME\", interval: \"PT1M\", begin: \"2024-10-10\", end: \"2024-10-11\"). ",
+          "Depending on the wording of the request, some arguments may be omitted. ",
+          "The begin, end, and interval args should be expressed in iso 8601 format. ",
+          "If the prompt doesn't contain an explicit interval, please choose a reasonable value depending on the duration between begin and end.",
+          "Interpret dates according to the trading schedule rather than calendar date whenever possible. ",
+          "In some prompts, the time window will be expressed in relative terms (e.g., \"last Monday\", or \"yesterday\") ",
+          "which must be translated to exact dates. To correctly interpret the dsl in the examples, assume the current time is 2024-10-11T12:00-05:00."
+        ],
+        "examples": [
+          {
+            "natural_language": "Get SP500 order imbalance from Wednesday.",
+            "dsl": "OrderImbalance(symbol: \"ES\", venue: \"CME\", begin: \"2024-10-08T17:00:00-06:00\", end: \"2024-10-09T16:00:00-06:00\", \"PT1M\")"
+          },
+          {
+            "natural_language": "Get SP500 order imbalance from Wednesday for the top 5 price levels.",
+            "dsl": "OrderImbalance(symbol: \"ES\", venue: \"CME\", begin: \"2024-10-08T17:00:00-06:00\", end: \"2024-10-09T16:00:00-06:00\", levels: 5, interval: \"PT1M\")"
+          },
+          {
+            "natural_language": "Get orderflow imbalance for 10 year T-Notes in the first minute after the market opened yesterday.",
+            "dsl": "OrderImbalance(symbol: \"ES\", venue: \"CME\", begin: \"2024-10-10T17:00:00-06:00\", end: \"2024-10-10T17:01:00-06:00\", interval: \"PT1S\")"
+          },
+          {
+            "natural_language": "Retrieve NVIDIA order flow from Oct 3",
+            "dsl": "OrderImbalance(symbol: \"NVDA\", venue: \"NASDAQ\", begin: \"2024-10-02TT18:00:00-05:00\", end: \"2024-10-03T17:00:00-05:00\", interval: \"PT1S\")"
+          },
+          {
+            "natural_language": "Show me order imbalance for Crude oil futures this week",
+            "dsl": "OrderImbalance(symbol: \"CL\", venue: \"CME\", begin: \"2024-10-06:T17:00:00-06:00\", interval: \"PT1H\")"
+          },
+          {
+            "natural_language": "Fetch hourly orderflow for Gold futures this week",
+            "dsl": "OrderImbalance(symbol: \"GC\", venue: \"CME\", begin: \"2024-10-06:T17:00:00-06:00\", interval: \"PT1H\")"
+          },
+          {
+            "natural_language": "What was the minute by minute orderflow for Tesla yesterday?",
+            "dsl": "OrderImbalance(symbol: \"TSLA\", venue: \"NASDAQ\", begin: \"2024-10-09T18:00:00-05:00\", end: \"2024-10-10T17:00:00-05:00\", interval: \"PT1M\")"
+          }
+        ]
+      },
+      {
+        "category": "VWAP",
+        "description": "This is a few shot learning example for an NL to DSL. The DSL function in question is VWAP (symbol_or_contract, datetime1, datetime2). The first argument is a future (like CL, for crude oil, or ES for SP 500) or a specific future contract like crude oil Dec 24 future (CLZ4) or SP 500 Dec 2024 (ESZ4). The second argument is where the VWAP has to be calculated (also the beginning of the time window), and the third argument is datetime specifying the end of the time window. So, for example, a 20-day VWAP on 9/30/2023, the first argument will be 9/30/2023, and the third argument will be 9/10/2023 (20 days prior). This will also have datetimes expressed as Today, Yesterday, Last Friday, Last Sunday, etc. For the purpose of this few-shot prompts, let's assume today is 10/11/2024.",
+        "prompt": "What was the VWAP for SP 500 Future over the past 30 days?",
+        "examples": [
+          {
+            "natural_language": "Find the 20 day VWAP of the SP future on 04/22/2023.",
+            "dsl": "VWAP(\"ES\", 4/22/2023, 4/02/2023)"
+          },
+          {
+            "natural_language": "Find Yesterday's 10 day VWAP of the Crude Oil Future.",
+            "dsl": "VWAP(\"CL\", 10/10/2024, 10/1/2024)"
+          },
+          {
+            "natural_language": "Find today's 30-day VWAP of the 10 year treasury future.",
+            "dsl": "VWAP(\"ZN\", 10/11/2024, 9/11/2024)"
+          },
+          {
+            "natural_language": "Find the 10 hour VWAP of the Natural Gas future from yesterday at 5:00 PM.",
+            "dsl": "VWAP(\"NG\", 9/27/2024 17:00:00, 9/27/2024 7:00:00)"
+          }
+        ]
+      },
+      {
+        "category": "Price Diff",
+        "description": "This is a few shot learning example for an NL to DSL. The DSL function in question is OHLCVPriceDiff (symbol_or_contract, datetime1, datetime2). The first argument is a future (like CL, for crude oil) or a specific future contract like crude oil Dec 24 future (CLZ4). The second and third arguments are dates, that the function finds the difference of prices at. Assume today is 10/15/2024",
+        "prompt": "What is the difference in  SP 500 Future from July 10th to July 14th?",
+        "examples": [
+          {
+            "natural_language": "What is the difference in ZN price from July 10th to July 14th?",
+            "dsl": "OHLCVPriceDiff(\"ZN\", 7/14/2024, 7/10/2024)"
+          },
+          {
+            "natural_language": "What is the price difference in SP Future price from Sept 10th and Sept 5th?",
+            "dsl": "OHLCVPriceDiff(\"ES\", 9/10/2024, 9/5/2024)"
+          },
+          {
+            "natural_language": "What is the price difference in Natural Gas Dec Future price from Aug 30th and Aug 20th?",
+            "dsl": "OHLCVPriceDiff(\"NGZ4\", 8/30/2024, 8/20/2024)"
+          }
+        ]
+      },
+      {
+        "category": "Absolute Volume",
+        "description": "this is a few shot learning example for an NL to DSL. The DSL function in question is AbsVolume(symbol, ContractType, datetime1, datetime2), which computes the highest volume in the date range, specified by datetime1, and datetime2. The first argument is a future symbol (like CL for crude oil, ES for SP 500 etc).  The second argument is the ContractType, which is \"Future\" for futures and \"FutureSpread\" for future spreads. the 3rd argument is the begin date and the 4th argument is the end date, between which the max volume is computed. If no future is specified use \"*\" from the first arugment. For this prompt, assume today is 10/11/2024 ",
+        "prompt": "Which  SP 500 Future traded the most between 7/14/2024 and 7/10/2024",
+        "examples": [
+          {
+            "natural_language": "Which SP Future traded the most between 7/14/2024 and 7/10/2024",
+            "dsl": "AbsVolume(\"ES\", \"Future\", 7/14/2024, 7/10/2024)"
+          },
+          {
+            "natural_language": "Which Gold Spread traded the most between Today and Sept 20th",
+            "dsl": "AbsVolume(\"GC\", \"FutureSpread\", 10/1/2024, 9/20/2024)"
+          },
+          {
+            "natural_language": "Which 10 year Spread traded the highest volume last week?",
+            "dsl": "AbsVolume(\"ZN\", \"FutureSpread\", 9/27/2024, 9/23/2024)"
+          },
+          {
+            "natural_language": "Which Corn Spread traded the most today?",
+            "dsl": "AbsVolume(\"ZC\", \"FutureSpread\", 10/1/2024, 10/1/2024)"
+          },
+          {
+            "natural_language": "Which Gold Future traded the most today?",
+            "dsl": "AbsVolume(\"GC\", \"Future\", 9/30/2024, 9/30/2024)"
+          }
+        ]
+      },
+      {
+        "category": "Roll",
+        "description": "This is a few shot learning example for an NL to DSL. The DSL function in question is FuturesRoll(symbol) which computes the rollover date for a future, ie when the highest volume swtches from one expiration to the next one. ",
+        "prompt": "When does the roll typically occur In  SP 500?",
+        "examples": [
+          {
+            "natural_language": "When does the roll typically occur In CL?",
+            "dsl": "FuturesRoll(\"CL\")"
+          },
+          {
+            "natural_language": "When does the roll typically occur for 10 year treasury?",
+            "dsl": "FuturesRoll(\"ZN\")"
+          },
+          {
+            "natural_language": "When does the roll usually occur for Gold Future",
+            "dsl": "FuturesRoll(\"GC\")"
+          },
+          {
+            "natural_language": "At what date does the future rollover happens for SP500?",
+            "dsl": "FuturesRoll(\"ES\")"
+          },
+          {
+            "natural_language": "When does the future rollover happens for the Corn Future?",
+            "dsl": "FuturesRoll(\"ZC\")"
+          }
+        ]
+      },
+      {
+        "category": "Daily Move",
+        "description": "This is a few shot learning example for an NL to DSL. ",
+        "prompt": "How often did Sp 500 Future settle + or - 10% over the previous day?",
+        "examples": [
+          {
+            "natural_language": "List all the days when the SP 500 closed + or -5% from the previous day's close",
+            "dsl": "ColDiff(filteredOHLCV(\"ES\"),\"ts_event\", -1, \"close\", \"close\", \"close_diff\", 5)"
+          },
+          {
+            "natural_language": "List all the days when the Crude Oil closed + or -2.5% from the previous day's close",
+            "dsl": "ColDiff(filteredOHLCV(\"CL\"),\"ts_event\", -1, \"close\", \"close\", \"close_diff\", 2.5)"
+          },
+          {
+            "natural_language": "Show me all the days when the Ten year Treaury Future opened + or -1% from the previous day's open",
+            "dsl": "ColDiff(filteredOHLCV(\"ZN\"),\"ts_event\", -1, \"open\", \"open\", \"open_diff\", 1.0)"
+          },
+          {
+            "natural_language": "List all the trading days when the Silver Future opened + or -1% from the previous day's close",
+            "dsl": "ColDiff(filteredOHLCV(\"SI\"),\"ts_event\", -1, \"open\", \"open\", \"open_diff\", 1.0)"
+          }
+        ]
+      },
+      {
+        "category": "Average Volume",
+        "description": "This is a few shot learning example for an NL to DSL. For the purpose of this few shot prompts, lets assume today is 10/3/2024 ",
+        "prompt": "When does the most open interest typically shift from one contract to another in SP 500 Future?",
+        "examples": [
+          {
+            "natural_language": "When does the most open interest typically shift from one contract to another in e-minis? ",
+            "dsl": "FuturesRoll(\"ES\")"
+          },
+          {
+            "natural_language": "Usually when does the open interest move from one contract to another in WTI?",
+            "dsl": "FuturesRoll(\"CL\")"
+          }
+        ]
+      },
+      {
+        "category": "Events",
+        "description": "This is a few shot learning example for an NL to DSL. In the NL prompt the user is asking to show the price of a future (treasury future, or SP 500 future etc) around an event type. The event type could be \"FOMC Announcement\" or \"Storage numbers\" etc. The DSL is as follows: EventsPriceAction(future_name, event_type, pre_event_window, post_event_window, number_of_events) first arg is the name of the future, (ex ES or ZN etc), the second argument is the event type, for example, \"FOMC announcement\", the 3rd argument is a time window for how much pre-event data to include in the end result, the 4th argument is how much post-event data to include in the end result. Both are strings of the form \"5d\" for 5 days, \"5h\" for 5 hours and \"120m\" for 120 minutes and so on. Usually, only one of the event periods is provided. Set the other to be half of what the user provided. The last argument is the number of events to consider. So if it’s 7, that means the most recent 7 events. If the user does not specify that, pass a large number.",
+        "prompt": "What did the volume and price of SP500 Future do following the release of the last 10 FOMC rates?",
+        "examples": [
+          {
+            "natural_language": "How did SP 500 Future do in the 5 days following the FOMC announcements?",
+            "dsl": "EventsPriceAction(\"ES\", \"FOMC Announcement\", \"2d\", \"5d\", 10)"
+          },
+          {
+            "natural_language": "How did the price of Treasury Future evolve in the 2 days following the Job Reports?",
+            "dsl": "EventsPriceAction(\"ZN\", \"Job Reports\", \"1d\", \"2d\", 10)"
+          },
+          {
+            "natural_language": "Show me the price movement and volume of Crude Oil futures during the 5-minute window following the last 5 OPEC meetings.",
+            "dsl": "EventsPriceAction(\"CL\", \"OPEC Meeting\", \"2m\", \"5m\", 5)"
+          },
+          {
+            "natural_language": "Show me the price movement and volume of SP 500 futures preceding the 10-minute window of the last 5 OPEC meetings.",
+            "dsl": "EventsPriceAction(\"CL\", \"OPEC Meeting\", \"10m\", \"5m\", 5)"
+          }
+        ]
+      },
+      {
+        "category": "Indicator, Standard Deviation",
+        "MulitLineDescription": [
+          "This is a few shot learning example for an NL to DSL. In the NL prompt the user is asking to show all the days when ",
+          "the price of a future (treasury future, or SP 500 future etc) moved more than m times the N days standard deviation, ",
+          "where both m and N are provided. For example, the DSL for \"Using a 90d standard deviation, give me the dates and ",
+          "amounts of the last 4+ standard deviation moves in WTI futures\" would be: ",
+          "r=standardDeviation(\"ES\", \"90d\")\n",
+          "r=math(r, \"abs_daily_change = Abs(close - open) comp = std_dev * 4\")\n",
+          "r=filter(r,\"abs_daily_change >= comp\"). ",
+          "The first line, creates a stream for StandardDeviation, based on the number of days, for example 90d standard deviation, ",
+          "and assigned it to r, the second line takes r from the first line and passes it to a math stream that creates 2 new columns, ",
+          "abs_daily_change and comp. comp is the factor (4, from 4+ in the prompt) times standard deviation. The third line filters ",
+          "(like a sql where) of rows where abs_daily_change >= comp"
+        ],
+        "prompt": "Using a 90d standard deviation, give me the dates and amounts of the last 1+ standard deviation moves in SP  futures ?",
+        "examples": [
+          {
+            "natural_language": "Using a 90d standard deviation, give me the dates and amounts of the last 1+ standard deviation moves in WTI futures ?",
+            "dsl": "r=standardDeviation(\"CL\", \"90d\")\nr=math(r, \"abs_daily_change = Abs(close - open) comp = std_dev * 1.0\")\nr=filter(r,\"abs_daily_change >= comp\")"
+          },
+          {
+            "natural_language": "Show me the times the SP Futures moved twice the 60 day standard deviation",
+            "dsl": "r=standardDeviation(\"ES\", \"60d\")\nr=math(r, \"abs_daily_change = Abs(close - open) comp = std_dev * 2.0\")\nr=filter(r,\"abs_daily_change >= comp\")"
+          },
+          {
+            "natural_language": "List all the days the Gold Futures moved 1.5 times the 45 day standard deviation",
+            "dsl": "r=standardDeviation(\"GC\", \"45d\")\nr=math(r, \"abs_daily_change = Abs(close - open) comp = std_dev * 1.5\")\nr=filter(r,\"abs_daily_change >= comp\")"
+          },
+          {
+            "natural_language": "List all the days the 10 year treasury future moved more than twice the 1 month standard deviation",
+            "dsl": "r=standardDeviation(\"ZN\", \"30d\")\nr=math(r, \"abs_daily_change = Abs(close - open) comp = std_dev * 2.0\")\nr=filter(r,\"abs_daily_change >= comp\")"
+          }
+        ]
+      }
+    ]
+  },
  "SparkApi": {
    "IsEnabled": false,
    "Credentials": {
@@ -77,7 +435,7 @@
        {
          "TagType": ".*",
          "Assembly": "DataStreamCore",
-         "Class": "bts.PubsubServerGenericDataHandler"
+          "Class": "bts.PubsubServerGenericDataHandler"
        }
      ]
    }
@@ -2,17 +2,26 @@ namespace Bts.DataStream;

 using BCG;

+using bts.api.core;
+
+using Bts.DataStreamCore;
+
+using Microsoft.Extensions.Logging;
+
 using Spark.Core.Disposables;

 using System;
+using System.Collections;
+using System.Collections.Generic;
+using System.Linq;
 using System.Reactive.Linq;

 public static class DataStreamExtensions {
   public static ViewStream AsViewStream(this IObservable<DataPacket> source) =>
      source as ViewStream ?? new ObservableViewStream(source);

-   public static ViewStream AsViewStream<T>(this IObservable<IDataPacket<T>> source) where T : IBlueRow =>
-      source as ViewStream ?? new ObservableViewStream(source.Select(dp => new DataPacket(dp.newRows, dp.addedRows, dp.deletedRows)));
+   public static ViewStream AsViewStream<T>(this IObservable<IDataPacket<T>> source, string? name = null, ILogger? logger = null) where T : IBlueRow =>
+      source as ViewStream ?? new ObservableViewStream(source.Select(dp => new DataPacket(dp.newRows, dp.addedRows, dp.deletedRows)), name, logger);

   public static IObservable<DataPacket> AsObservable(this ViewStream source) {
      return source as IObservable<DataPacket> ?? Observable.Create<DataPacket>(Subscribe);
@@ -26,4 +35,100 @@ public static class DataStreamExtensions {
               $"Expected DataPacket, got {newValue?.GetType()}", nameof(newValue)));
      }
   }
+
+   public static IObservable<IDataPacket<T>> SelectDataPacket<T>(this IObservable<IDataPacket> source, Func<IBlueRow, T> selector) where T : IBlueRow =>
+      source.Select(dp => {
+         dp.get_new_added_deleted(out var newRows, out var addedRows, out var deletedRows);
+         return new DataPacket<T>(CastRows(newRows), CastRows(addedRows), CastRows(deletedRows));
+
+         IBlueRowCollection<T> CastRows(ICollection rows) =>
+            new BlueTableSelectorWrapper<T>(
+               rows as IBlueRowCollection ?? new BlueRowCollection(rows, dp.Schema), selector);
+      });
+
+   public static IObservable<TableUpdate<IBlueRow, int>> AsTables(this IObservable<IDataPacket> source) =>
+      source.AsTables<IBlueRow, int>(r => r, r => r.Id());
+
+   public static IObservable<TableUpdate<T, int>> AsTables<T>(this IObservable<IDataPacket> source, Func<IBlueRow, T> valueSelector) where T : IBlueRow =>
+      source.AsTables<T, int>(valueSelector, r => r.Id());
+
+   public static IObservable<TableUpdate<T, int>> AsTables<T>(
+      this IObservable<IDataPacket> source, Func<IBlueRow, T> valueSelector, Func<T, int> keySelector) =>
+      source.AsTables<T, int>(valueSelector, keySelector);
+
+   public static IObservable<TableUpdate<T, TKey>> AsTables<T, TKey>(
+      this IObservable<IDataPacket> source, Func<IBlueRow, T> valueSelector, Func<T, TKey> keySelector, Func<IBlueRow, TKey>? rowKeySelector = null) where TKey : IEquatable<TKey> {
+      rowKeySelector ??= r => keySelector(valueSelector(r));
+      return source.Scan(TableUpdate<T, TKey>.mk_empty(keySelector), (res, dp) => {
+         var bd = res.to_builder();
+         dp.get_new_added_deleted(out var newRows, out var addedRows, out var deletedRows);
+         foreach (IBlueRow r in addedRows) {
+            bd.add_or_update(valueSelector(r));
+         }
+         foreach (IBlueRow r in deletedRows) {
+            bd.remove(rowKeySelector(r));
+         }
+         return bd.to_table();
+      });
+   }
+
+   public static IObservable<IDataPacket<T>> AsDataPackets<T>(this IObservable<TableUpdate<T, int>> source, IBlueTableSchema schema) where T : IBlueRow =>
+      source.Scan(
+         (Table: new BlueTableImmutable<T>(schema), Dp: (IDataPacket<T>)DataPacket<T>.Empty(schema)),
+         (res, table) => {
+            var bd = res.Table.ToDataPacketBuilder();
+            foreach (var r in table.removed) {
+               bd.Remove(r.Id());
+            }
+            foreach (var r in table.added) {
+               bd.AddOrUpdate(r);
+            }
+            var resDp = bd.ToDataPacketT(out var newRows, out var addedRows, out var deletedRows);
+            return (newRows, resDp);
+         })
+         .Where(t => t.Dp.addedRows.Any() || t.Dp.deletedRows.Any())
+         .Select(t => t.Dp);
+
+   public static ViewStream AsViewStream<T>(this IObservable<TableUpdate<T, int>> source, IBlueTableSchema schema) where T : IBlueRow =>
+      source.AsDataPackets(schema).AsViewStream();
+
+   public static int ResolveIndices(this IBlueTableSchema schema, string name) {
+      var indices = ResolveIndices(schema, new[] { name });
+      return indices[0];
+   }
+
+   public static (int, int) ResolveIndices(this IBlueTableSchema schema, string name1, string name2) {
+      var indices = ResolveIndices(schema, new[] { name1, name2 });
+      return (indices[0], indices[1]);
+   }
+
+   public static (int, int, int) ResolveIndices(this IBlueTableSchema schema, string name1, string name2, string name3) {
+      var indices = ResolveIndices(schema, new[] { name1, name2, name3 });
+      return (indices[0], indices[1], indices[2]);
+   }
+
+   public static (int, int, int, int) ResolveIndices(this IBlueTableSchema schema, string name1, string name2, string name3, string name4) {
+      var indices = ResolveIndices(schema, new[] { name1, name2, name3, name4 });
+      return (indices[0], indices[1], indices[2], indices[3]);
+   }
+
+   public static int[] ResolveIndices(this IBlueTableSchema schema, string[] columnNames) {
+      var columns = schema.columnInfo;
+      var indices = new int[columnNames.Length];
+      int nResolved = 0;
+      for (int i = 0; i < columnNames.Length; i++) {
+         int origNResolved = nResolved;
+         for (int j = 0; j < columns.Length; j++) {
+            if (columns[j].colName == columnNames[i]) {
+               indices[i] = j;
+               nResolved++;
+               break;
+            }
+         }
+         if (nResolved == origNResolved) {
+            throw new ArgumentException($"No column named '{columnNames[i]}'", nameof(schema));
+         }
+      }
+      return indices;
+   }
 }
@@ -3,24 +3,25 @@ namespace bts.SilverPine {
   using BCG;
   using System.Collections.Generic;
   using Bts.DataStreamExpressions;
+   using Bts.DataStream;

   // All this does is take in a dsl expression, parse it and fire the data that come out of running it.
   // So for example, if an expresison is passed 
   [DataStreamUserRespresentation("dsl")]
   public class DSLStream : ViewStream {
      private readonly string _dsl;
-      public DSLStream(string dsl) {
+      public DSLStream(string dsl, IDataStreamFactory streamFactory) {
         Log.Gen.info($"DSLStream.cons dsl: {dsl}");
         _dsl = dsl;
-         ViewStream rs = build_stream(dsl);
-         rs = rs ?? new LiteralStream("error", $"msg:invalid dsl \"{dsl}\"");
+         var rs = build_stream(streamFactory, dsl);
+         rs = rs ?? new LiteralStream("error", $"msg:invalid dsl");
         mySetRealSource(rs);
      }

-      static ViewStream? build_stream(string stream_expression) {
+      static ViewStream? build_stream(IDataStreamFactory streamFactory, string stream_expression) {
         try {
            var exp = DataStreamExpressionsInterpreter.Evaluate(stream_expression);
-            var rv = exp.getView(new Dictionary<string, StatementExpression>(0));
+            var rv = exp.getView(new Dictionary<string, StatementExpression>(0), streamFactory);
            return rv;

         } catch (Exception ex) {
@@ -29,4 +30,4 @@ namespace bts.SilverPine {
         }
      }
   }
-}
+}
@@ -4,14 +4,21 @@ using BCG;

 using bts;

+using Microsoft.Extensions.Logging;
+
 using System;

 public class ObservableViewStream : ViewStream, IObservable<DataPacket>, IDisposable {
   readonly IObservable<DataPacket> source;
+   readonly string? name;
+   readonly ILogger? logger;
   IDisposable? subscription;

-   public ObservableViewStream(IObservable<DataPacket> source) =>
+   public ObservableViewStream(IObservable<DataPacket> source, string? name = null, ILogger? logger = null) {
      this.source = source;
+      this.name = name;
+      this.logger = logger;
+   }

   public void Dispose() {
      GC.SuppressFinalize(this);
@@ -23,7 +30,13 @@ public class ObservableViewStream : ViewStream, IObservable<DataPacket>, IDispos

   protected override void myOnFirstSubscriber() {
      Log.isnull(subscription);
-      subscription = source.Subscribe(myFireEvent);
+      subscription = source.Subscribe(myFireEvent, OnError, OnCompleted);
+
+      void OnError(Exception exception) =>
+         logger?.LogError("Error in ObservableViewStream. name: {Name}, error: {Exception}", name, exception);
+
+      void OnCompleted() =>
+         logger?.LogInformation("ObservableViewStream completed. name: {Name}", name);
   }

   protected override void myOnLastSubscriber() {
@@ -34,12 +34,12 @@ namespace BCG {
            view_publisher = view_publisher_;
         }
      }
-      static ViewStream? build_stream(string stream_expression)
+      static ViewStream? build_stream(IDataStreamFactory streamFactory, string stream_expression)
      {
         try
         {
            var exp = DataStreamExpressionsInterpreter.Evaluate(stream_expression);
-            var rv = exp.getView(new Dictionary<string, StatementExpression>(0));
+            var rv = exp.getView(new Dictionary<string, StatementExpression>(0), streamFactory);
            return rv;

         } catch (Exception ex) {
@@ -51,15 +51,17 @@ namespace BCG {
         _stream_pub_map = new Dictionary<string, StreamPubUnit>();
      readonly IPubTableBroker _broker;
      readonly IManagedSubscriber _sub;
+      readonly IDataStreamFactory streamFactory;
      readonly object syncRoot = new();
      IDisposable? cloudStreamsSub;
      IDisposable? remoteRequestsSub;
      bool isDisposed;


-      public RemoteQueryManager(IManagedSubscriber managedSubscriber, IPubTableBroker pubTableBroker) {
+      public RemoteQueryManager(IManagedSubscriber managedSubscriber, IPubTableBroker pubTableBroker, IDataStreamFactory streamFactory) {
         _sub = managedSubscriber;
         _broker = pubTableBroker;
+         this.streamFactory = streamFactory;
      }

      public Task StartAsync(CancellationToken cancellationToken) {
@@ -102,7 +104,11 @@ namespace BCG {

      private void _spawn_remote_stream(string name, string stream_expression, bool persist) {
         bts.Log.Gen.info($"CloudStreamManager.assign name: {name} stream_expression: {stream_expression}");
-         ViewStream stream = build_stream(stream_expression);
+         var stream = build_stream(streamFactory, stream_expression);
+         if (stream is null) {
+            Log.DataFlow.error($"Failed to spawn remote stream. name: {name}; expression: {stream_expression}");
+            return;
+         }
         _expose_to_ws(name, stream);
         if (persist) {
            stream = new PersistorStream(stream, name, "C:\\Temp\\", 1000000);   // 10MBs
@@ -4,6 +4,8 @@ namespace bts.SilverPine {
   using System;

   public class NullFilterStream : ViewStream {
+      public NullFilterStream(ViewStream parent_str, string col_name)
+      : this(parent_str, col_name, true) { }
      public NullFilterStream(ViewStream parent_str, string col_name, bool filter_out_doublenans = false) {
         ViewStream rs = new FilterStream(parent_str,
         r => {
@@ -0,0 +1,91 @@
+namespace bts.SilverPine {
+   using BCG;
+
+   using System;
+   using System.Collections.Generic;
+   using System.Linq;
+
+   using ZstdSharp.Unsafe;
+
+   public enum GroupByBundleOp {
+      Mean,
+      Median,
+      Mode,
+      StdDev,
+      Max,
+      Min,
+      First,
+      Last,
+      Sum
+   }
+   public class GroupByBundle {
+      public readonly List<Tuple<string, GroupByBundleOp>> operations;
+
+      public static GroupByBundle parse(string text) {
+         // text is inp_col: op, inp_col: op, inp_col:op
+         List<Tuple<string, GroupByBundleOp>> rv = new List<Tuple<string, GroupByBundleOp>>();
+         string[] calcs = text.Split(',');
+         foreach (string c in calcs) {
+            string[] toks = c.Split(':');
+            string inp_col = toks[0].Trim();
+            string op_txt = toks[1].Trim();
+            GroupByBundleOp op = Enum.Parse<GroupByBundleOp>(op_txt, true);
+            rv.Add(Tuple.Create(inp_col, op));
+         }
+         return new GroupByBundle(rv);
+      }
+
+      public GroupByBundle(List<Tuple<string, GroupByBundleOp>> operations_) {
+         operations = operations_;
+      }
+
+      public static IBlueRow group_by(List<IBlueRow> rows, GroupByBundle parameters) {
+         if (rows.Count() == 0) {
+            return null;
+         }
+
+         IBlueRow ir = rows.ElementAt(0);
+         IBlueTableSchema in_schema = ir.schema;
+         IBlueTableSchema out_schema = in_schema;
+         IDictionary<string, Tuple<string, GroupByBundleOp>> out_col_map =
+            new Dictionary<string, Tuple<string, GroupByBundleOp>>();
+         foreach (var t in parameters.operations) {
+            string out_col_name = $"{t.Item1}_{t.Item2}".ToLower();
+            out_schema = out_schema.add_column(TypeCode.Double, out_col_name);
+            out_col_map[out_col_name] = t;
+         }
+         MutableBlueRow or = new MutableBlueRow(out_schema, ir);
+         foreach (var kv in out_col_map) {
+            string out_col_name = kv.Key;
+            string in_col_name = kv.Value.Item1;
+            GroupByBundleOp op = kv.Value.Item2;
+            IEnumerable<double> col_vals = BlueDataUtility.get_colvalues<double>(rows, in_col_name);
+            double comp_res = compute_op(col_vals, op);
+            or.setValue(out_col_name, comp_res);
+         }
+         return or;
+      }
+
+      public static double compute_op(IEnumerable<double> col_vals, GroupByBundleOp op) {
+         switch (op) {
+            case GroupByBundleOp.Max:
+               return col_vals.Max<double>();
+            case GroupByBundleOp.Min:
+               return col_vals.Min<double>();
+            case GroupByBundleOp.Mean:
+               return col_vals.Sum() / col_vals.Count();
+            case GroupByBundleOp.Sum:
+               return col_vals.Sum();
+            case GroupByBundleOp.First:
+               return col_vals.First();
+            case GroupByBundleOp.Last:
+               return col_vals.Last();
+            default:
+               // TODO
+               Log.assert(false);
+               return Double.NaN;
+         }
+      }
+
+   }
+}
@@ -2,10 +2,13 @@
 using System.Collections.Generic;
 using System.Diagnostics;
 using System.Linq;
+using System.Text.Json;

 using bts;
+using bts.SilverPine;

 namespace BCG {
+   [DataStreamUserRespresentation("aggregate")]
   public class NonIncGroupByStream : ViewStream {
      private class DelNonIncGroupByEvaluator : INonIncGroupByEvaluator {
         Func<string[]> _groupby_cols_func;
@@ -43,13 +46,31 @@ namespace BCG {
         _main_stream = main_stream;
         _groupby_func = groupby_func;
         _groupby_cols = _groupby_func.get_group_by_cols();
-         _row_hasher = new BlueRowHasher(_groupby_cols.Length);
+         _row_hasher = new BlueRowHasher(_groupby_cols == null ? 0 : _groupby_cols.Length);
      }

      public NonIncGroupByStream(ViewStream main_stream, Func<string[]> groupby_cols_func,
                                 Func<List<IBlueRow>, List<IBlueRow>> evaluate_func)
                                 : this(main_stream, new DelNonIncGroupByEvaluator(groupby_cols_func, evaluate_func)) { }

+      public NonIncGroupByStream(ViewStream parent_str, string groupby_col, string standard_group_by_text, string optional_sort_col = null) {
+         GroupByBundle std_group_by = GroupByBundle.parse(standard_group_by_text);
+         Func<string[]> groupby_cols_func = () => {
+            return String.IsNullOrEmpty(groupby_col) ? null : new string[] { groupby_col };
+         };
+         Func<List<IBlueRow>, List<IBlueRow>> evaluate_func = (group_rows) => {
+            if (!String.IsNullOrEmpty(optional_sort_col))
+               group_rows.Sort((r1, r2) => BlueDataUtility.compare_rows(r2, r1, optional_sort_col));
+            IBlueRow rv = GroupByBundle.group_by(group_rows, std_group_by);
+            return new IBlueRow[] { rv }.ToList();
+         };
+         ViewStream rs = new NonIncGroupByStream(parent_str, groupby_cols_func, evaluate_func);
+         mySetRealSource(rs);
+      }
+      public NonIncGroupByStream(ViewStream parent_str, string standard_group_by_text) :
+      this(parent_str, String.Empty, standard_group_by_text) {
+      }
+

      protected override void myOnFirstSubscriber() { _main_stream.onChange += _main_stream_onchange; }

@@ -83,7 +104,9 @@ namespace BCG {
            Log.DataFlow.info(
               $"Time to evaluate {grouped_rows.Count} groups {_timer.ElapsedMilliseconds}ms, ratio {_n_expensive_fires}/{_n_total_fires}");
         }
-         myFireEvent(new DataPacket(total_state, total_state, myLastValue == null ? new BlueTable(total_state.schema) : (myLastValue as DataPacket).newRows));
+         // TODO,
+         if (total_state != null)
+            myFireEvent(new DataPacket(total_state, total_state, myLastValue == null ? new BlueTable(total_state.schema) : (myLastValue as DataPacket).newRows));
      }
   }
 }
@@ -1,10 +1,12 @@
 using bts;
+using bts.collections;

 using mbts;

 using System;
 using System.Collections.Generic;
 using System.Diagnostics;
+using System.Linq;
 using System.Security.Cryptography;

 namespace BCG {
@@ -35,6 +37,38 @@ namespace BCG {
         _is_tag_along = _join_spec.join_col_names.Length == 0;
      }

+      private bool _filled_missing_data = false;
+      private void _fill_missing_data(DataPacket dp) {
+         if (_filled_missing_data)
+            return;
+         Func<string[], bool> null_or_empty = (str_arr) => {
+            if (str_arr == null || str_arr.Length == 0)
+               return true;
+            foreach (string s in str_arr) {
+               if (!String.IsNullOrEmpty(s)) {
+                  return false;
+               }
+            }
+            return true;
+         };
+         Func<string[], string[]> remove_bad_cols = (col_names) => {
+            HashSet<string> schema_col_map = new HashSet<string>(dp.schema.columnNames);
+            List<string> cols_list = new List<string>(col_names);
+            cols_list.RemoveWhere(c => !schema_col_map.Contains(c));
+            return cols_list.ToArray();
+         };
+         if (null_or_empty(_join_spec.data_col_names)) {
+            _join_spec.data_col_names = dp.schema.columnNames;
+         }
+         _join_spec.data_col_names = remove_bad_cols(_join_spec.data_col_names);
+         if (null_or_empty(_join_spec.data_col_names)) {
+            _join_spec.data_col_names = dp.schema.columnNames;
+         }
+         if (null_or_empty(_join_spec.projected_col_names)) {
+            _join_spec.projected_col_names = _join_spec.data_col_names;
+         }
+      }
+
      public void init() {
         _join_spec.stream.onChange += _on_stream_cb;
      }
@@ -43,6 +77,7 @@ namespace BCG {
         Log.Gen.info($"NonIncJoinBookkeeper._on_stream_cb dp_obj: {dp_obj}");
         var timer = Stopwatch.StartNew();
         DataPacket dp = (DataPacket)dp_obj;
+         _fill_missing_data(dp);
         if (_schema == null) {
            _schema = dp.schema;
            _join_col_indices = BlueDataUtility.get_column_indices(_schema, _join_spec.join_col_names);
@@ -7,11 +7,12 @@ using System.Threading.Tasks;
 using mbts;

 namespace BCG {
+   // this a collection of data, theres no joining logic in here
   public class NonIncJoinSpec {
      public ViewStream stream;
      // join col names are the names on which the streams are joined (for equality),
      // data col names are the names in the original stream
-      // projected col naes are their new names, ie if its x:l_x, then x is in the 
+      // projected col names are their new names, ie if its x:l_x, then x is in the 
      // data_col_names and l_x in projected cols
      public string[] join_col_names, data_col_names, projected_col_names;

@@ -1,9 +1,8 @@
-using Google.Protobuf.WellKnownTypes;
-
 using System;
-using BCG.Utilities.General;

-namespace BCG {
+using BCG;
+
+namespace bts.SilverPine {
   // Keeps adding changing rows to the stream, hence preserving the history
   // and creating a time series, for example this can be used to make bbo updates
   // into a time series of prices/qty 
@@ -0,0 +1,54 @@
+namespace Bts.DataStream;
+
+using BCG;
+
+using System;
+using System.Collections.Generic;
+using System.Data;
+using System.Data.SQLite;
+using System.Linq;
+using System.Threading;
+
+public class SQLiteEnumerable : IAsyncEnumerable<IBlueRowCollection> {
+   readonly string location;
+   readonly string statement;
+   readonly int pageSize;
+
+   public SQLiteEnumerable(string location, string statement, int pageSize = 65536) {
+      this.location = location;
+      this.statement = statement;
+      this.pageSize = pageSize;
+   }
+
+   public async IAsyncEnumerator<IBlueRowCollection> GetAsyncEnumerator(CancellationToken cancellationToken = default) {
+      var dataTable = new DataTable();
+      await using var conn = new SQLiteConnection($"Data Source={location};Version=3");
+      await conn.OpenAsync(cancellationToken);
+      await using var cmd = new SQLiteCommand(statement, conn);
+      await using var reader = cmd.ExecuteReader();
+      var schema = await reader.GetColumnSchemaAsync(cancellationToken);
+      var columns = schema.OrderBy(c => c.ColumnOrdinal ?? int.MaxValue)
+         .Select((c, i) => new ColumnInfo(c.ColumnName, c.ColumnOrdinal ?? i, c.DataType))
+         .Append(new ColumnInfo("id", schema.Count, typeof(int)))
+         .ToArray();
+      var blueSchema = new BlueTableSchema("anon", columns);
+      BlueTable? table = null;
+      int rowId = 1;
+      while (await reader.ReadAsync(cancellationToken)) {
+         table ??= new BlueTable(blueSchema);
+         var rowVals = new object[blueSchema.columnInfo.Length];
+         if (reader.GetValues(rowVals) != rowVals.Length - 1) {
+            throw new InvalidOperationException("Sqlite reader value count for single row not equal to column count");
+         }
+         rowVals[^1] = rowId++;
+         table.addRow(new BlueRow(blueSchema, rowVals));
+         if (table.Count == pageSize) {
+            yield return table;
+            table = null;
+         }
+      }
+      if (table is not null) {
+         yield return table;
+      }
+   }
+}
@@ -9,161 +9,16 @@ using BCG;
 using System.Linq;
 using System.Data.Entity;
 using System.Collections.Generic;
+using System.Data.SQLite;
+

 namespace bts.SilverPine {

-   public class DataRowWrapperRow : IBlueRow {
-      private readonly DataRow _row;
-      private readonly IBlueTableSchema _schema;
-      private readonly int _id;
-      public DataRowWrapperRow(IBlueTableSchema schema, DataRow row, int id) {
-         _row = row;
-         _id = id;
-         _schema = schema;
-      }
-      public virtual IBlueTableSchema schema { get { return _schema; } }
-
-      public virtual object this[string col_name] {
-         get {
-            ColumnInfo ci = _schema[col_name];
-            return this[ci.colNo];
-         }
-      }
-      public virtual object this[int index] {
-         get {
-            if (_schema[index].colName == "id")
-               return _id;
-            return _row[index];
-         }
-      }
-
-      public virtual int Count { get { return _row.ItemArray.Length; } }
-
-      public virtual bool IsSynchronized { get { return false; } }
-
-      public virtual object SyncRoot { get { return this; } }
-
-      public virtual void CopyTo(Array array, int index) { _row.ItemArray.CopyTo(array, index); }
-
-      public virtual bool IsFixedSize { get { return true; } }
-
-      virtual public bool IsReadOnly { get { return true; } }
-
-      /// <summary>
-      /// The key can either be the string corrsponding to the column name or
-      /// the order of the column
-      /// </summary>
-      public virtual object this[object key] {
-         get {
-            if (key is string)
-               return this[(string)key];
-            else if (key is int)
-               return this[(int)key];
-            throw new BlueDataException("Wrong type while indexing the row");
-         }
-         set { throw new NotSupportedException("The IBlueRow is readonly"); }
-      }
-
-      public virtual ICollection Keys { get { return _schema.columnNames; } }
-
-      public virtual ICollection Values { get { return _row.ItemArray; } }
-
-      public virtual void Add(object key, object value) { throw new NotSupportedException("The IBlueRow is readonly"); }
-
-      public virtual void Clear() { throw new NotSupportedException("The IBlueRow is readonly"); }
-
-      public virtual bool Contains(object key) { return _schema[(string)key] != null; }
-
-      public IDictionaryEnumerator GetEnumerator() { return new BlueRowDictionaryEnumerator(this); }
-
-      IEnumerator IEnumerable.GetEnumerator() { return new BlueRowEnumerator(this); }
-
-      public virtual void Remove(object key) { throw new NotSupportedException("The IBlueRow is readonly"); }
-
-   }
-
-   public class DataTableWrapperTable : IBlueRowCollection {
-
-      public class DataTableWrapperTableEnumerator : IEnumerator {
-         private IEnumerator _enumerator;
-         private IBlueTableSchema _schema;
-         private int _id = 0;
-         public DataTableWrapperTableEnumerator(DataTable table) {
-            _enumerator = table.Rows.GetEnumerator();
-            _schema = BlueDataUtility.getSchema(table);
-
-         }
-         public object Current {
-            get {
-               DataRow r = _enumerator.Current as DataRow;
-               return new DataRowWrapperRow(_schema, r, _id);
-            }
-         }
-         public bool MoveNext() {
-            _id++;
-            return _enumerator.MoveNext();
-         }
-         public void Reset() {
-            _id = 0;
-            _enumerator.Reset();
-         }
-      }
-
-      private readonly DataTable _table;
-      private IBlueTableSchema _schema;
-      private IEnumerator _enumerator;
-
-      public DataTableWrapperTable(DataTable table) {
-         _table = table;
-         _schema = BlueDataUtility.getSchema(_table);
-         _enumerator = new DataTableWrapperTableEnumerator(table);
-      }
-      public virtual IBlueRow this[int id] {
-         get {
-            // its assumed the rows index == id
-            DataRow dr = _table.Rows[id];
-            return new DataRowWrapperRow(_schema, dr, id);
-         }
-         set {
-
-         }
-      }
-
-      public virtual IBlueTableSchema schema { get { return _schema; } }
-
-      public virtual bool containsID(int id) { return id < _table.Rows.Count; }
-
-      // private List<IBlueRow> _rows() {
-      //    List<IBlueRow> rv = new List<IBlueRow>();
-      //    int id = 0;
-      //    foreach (DataRow r in _table.Rows) {
-      //       rv.Add(new DataRowWrapperRow(r, id++));
-      //    }
-      //    return rv;
-      // }
-
-      public virtual IEnumerator GetEnumerator() {
-         // var rows = _rows();
-         // return rows.GetEnumerator();
-         return _enumerator;
-      }
-
-      public virtual bool IsSynchronized { get { return false; } }
-
-      public virtual object SyncRoot { get { return this; } }
-
-      public virtual void CopyTo(Array array, int index) {
-         Log.assert(false);
-         //_rows().CopyTo(array as IBlueRow[], index); 
-      }
-
-      public virtual int Count {
-         get { return _table.Rows.Count; }
-      }
-   }
-
   public static class SQLiteHelper {
-      public static DataTable ExecuteSelectQuery(string connection_string, string query) {
+      public static string location = "C:\\Users\\m_tah\\development\\data\\";
+      public static string db_name = "stream_data.db";
+
+      public static DataTable execute_select_query(string connection_string, string query) {
         // Create a new DataTable to store the results
         DataTable dataTable = new DataTable();

@@ -183,48 +38,85 @@ namespace bts.SilverPine {
               Console.WriteLine("An error occurred: " + ex.Message);
            }
         }
-
         // Return the populated DataTable
         return dataTable;
      }
-   }

-   [DataStreamUserRespresentation("SQLite")]
-   public class SQLiteStream : ViewStream {
-      private readonly string _connection_string, _query;
-      private Stopwatch _stopwatch = new Stopwatch();
-      public SQLiteStream(string connection_string, string query) {
-         _connection_string = connection_string;
-         _query = query;
+      public static List<Dictionary<string, object>> ExecuteBulkRead(string connectionString, string query) {
+         var result = new List<Dictionary<string, object>>();
+
+         using (var connection = new SQLiteConnection(connectionString)) {
+            connection.Open();
+
+            // Begin a transaction to optimize read speed
+            using (var transaction = connection.BeginTransaction())
+            using (var command = new SQLiteCommand(query, connection, transaction))
+            using (var reader = command.ExecuteReader()) {
+               while (reader.Read()) {
+                  var row = new Dictionary<string, object>();
+
+                  for (int i = 0; i < reader.FieldCount; i++) {
+                     string columnName = reader.GetName(i);
+                     object columnValue = reader.IsDBNull(i) ? null : reader.GetValue(i);
+                     row[columnName] = columnValue;
+                  }
+
+                  result.Add(row);
+               }
+
+               // Commit the transaction
+               transaction.Commit();
+            }
+         }
+
+         return result;
      }

+      public static List<Dictionary<string, object>> get_table(string connectionString, string query) {
+         var result = new List<Dictionary<string, object>>();

-      private void _read_from_sqlite_nd_fire() {
-         _stopwatch.Start();
-         DataTable dt = SQLiteHelper.ExecuteSelectQuery(_connection_string, _query);
-         _stopwatch.Stop();
-         Log.Gen.info($"SQLiteStream._read_from_sqlite_and_fire time to read db: {_stopwatch.ElapsedMilliseconds} msecs");
-         _stopwatch.Start();
-         IBlueRowCollection rows = BCG.BlueDataUtility.toIBlueRowCollection(dt);
-         //IBlueRowCollection rows = new DataTableWrapperTable(dt);
-         _stopwatch.Stop();
-         Log.Gen.info($"SQLiteStream._read_from_sqlite_and_fire time to load into IBlueRowCollection: {_stopwatch.ElapsedMilliseconds} msecs");
-         DataPacket dp = BlueDataUtility.build_dp_inc(rows, myLastValue as DataPacket);
-         myFireEvent(dp);
+         using (var connection = new SQLiteConnection(connectionString)) {
+            try {
+               connection.Open();
+
+               using (var command = new SQLiteCommand(query, connection))
+               using (var reader = command.ExecuteReader()) {
+                  while (reader.Read()) {
+                     var row = new Dictionary<string, object>();
+
+                     for (int i = 0; i < reader.FieldCount; i++) {
+                        string columnName = reader.GetName(i);
+                        object columnValue = reader.IsDBNull(i) ? null : reader.GetValue(i);
+                        row[columnName] = columnValue;
+                     }
+
+                     result.Add(row);
+                  }
+               }
+            } catch (Exception ex) {
+               Console.WriteLine("An error occurred: " + ex.Message);
+            }
+         }
+         return result;
      }

-      protected override void myOnFirstSubscriber() {
-         _read_from_sqlite_nd_fire();
+      public static IEnumerable<Dictionary<string, object>> get_table_2(string connectionString, string query) {
+         using (var connection = new SQLiteConnection(connectionString)) {
+            connection.Open();
+            using (var command = new SQLiteCommand(query, connection))
+            using (var reader = command.ExecuteReader()) {
+               while (reader.Read()) {
+                  var row = new Dictionary<string, object>();
+                  for (int i = 0; i < reader.FieldCount; i++) {
+                     string columnName = reader.GetName(i);
+                     object columnValue = reader.IsDBNull(i) ? null : reader.GetValue(i);
+                     row[columnName] = columnValue;
+                  }
+                  yield return row;
+               }
+            }
+         }
      }
-
-      protected override void myOnLastSubscriber() {
-         // this value will be cleared by the base class
-         // but in this particular stream this behaviour
-         // is not usefull since any new subscribers 
-         // would force the query to be recreated
-         //myLastDP = myLastValue;
-      }
-
   }
 }

@@ -0,0 +1,73 @@
+namespace bts.SilverPine {
+   using System;
+   using System.Data;
+   using System.Data.SQLite;
+   using System.Diagnostics;
+   using System.Collections;
+
+
+   using BCG;
+
+   [DataStreamUserRespresentation("SQLite")]
+   public class SQLiteStream : ViewStream {
+      private readonly string _connection_string, _query;
+      private Stopwatch _stopwatch = new Stopwatch();
+
+      private SQLiteStream(string connection_string, string query, bool noop) {
+         _connection_string = connection_string;
+         _query = query;
+      }
+
+      public SQLiteStream(string db_name, string query)
+      :
+      this($"Data Source={SQLiteHelper.location}{db_name};Version=3", query, true) { }
+
+      public SQLiteStream(string query)
+      :
+      this(SQLiteHelper.db_name, query) {
+      }
+
+
+      private void _read_from_sqlite_and_fire() {
+         _stopwatch.Start();
+         DataTable dt = SQLiteHelper.execute_select_query(_connection_string, _query);
+         _stopwatch.Stop();
+         Log.Gen.info($"SQLiteStream.ExecuteSelectQuery time to read db: {_stopwatch.ElapsedMilliseconds} msecs");
+         _stopwatch.Restart();
+         #region perf measures
+         // SQLiteHelper.get_table(_connection_string, _query);
+         // _stopwatch.Stop();
+         // Log.Gen.info($"SQLiteStream.get_table time to read db: {_stopwatch.ElapsedMilliseconds} msecs");
+         // _stopwatch.Restart();
+         // SQLiteHelper.ExecuteBulkRead(_connection_string, _query);
+         // _stopwatch.Stop();
+         // Log.Gen.info($"SQLiteStream.ExecuteBulkRead time to read db: {_stopwatch.ElapsedMilliseconds} msecs");
+         // _stopwatch.Restart();
+         // SQLiteHelper.get_table_2(_connection_string, _query);
+         // _stopwatch.Stop();
+         // Log.Gen.info($"SQLiteStream.get_table_2 time to read db: {_stopwatch.ElapsedMilliseconds} msecs");
+         // _stopwatch.Restart();
+         #endregion
+         IBlueRowCollection rows = BCG.BlueDataUtility.toIBlueRowCollection(dt);
+         //IBlueRowCollection rows = new DataTableWrapperTable(dt);
+         _stopwatch.Stop();
+         Log.Gen.info($"SQLiteStream._read_from_sqlite_and_fire time to load into IBlueRowCollection: {_stopwatch.ElapsedMilliseconds} msecs");
+         DataPacket dp = BlueDataUtility.build_dp_inc(rows, myLastValue as DataPacket);
+         myFireEvent(dp);
+      }
+
+      protected override void myOnFirstSubscriber() {
+         _read_from_sqlite_and_fire();
+      }
+
+      protected override void myOnLastSubscriber() {
+         // this value will be cleared by the base class
+         // but in this particular stream this behaviour
+         // is not usefull since any new subscribers 
+         // would force the query to be recreated
+         //myLastDP = myLastValue;
+      }
+
+   }
+
+}
@@ -80,7 +80,7 @@ namespace BCG {

      public IBlueRow map(IBlueRow row) {
         MutableBlueRow rv = new MutableBlueRow(getOutputSchema(row.schema), row);
-         rv.setValue(myConversionColName, mapDateTime(row[myConversionColName], myConversion));
+         rv.setValue(myConversionColName, mapDateTime(row.get_datetime(myConversionColName), myConversion));
         return rv;
         //return BlueDataUtility.getNewReplacedRow(row, mapDateTime(row[myConversionColName], myConversion), myConversionColName);
      }
@@ -10,9 +10,11 @@ namespace bts.SilverPine {
         Func<IBlueTableSchema, IBlueTableSchema> get_schema = (s) => {
            IBlueTableSchema rv = s.add_column(System.TypeCode.DateTime, $"{inp_col}_date");
            rv = rv.add_column(System.TypeCode.String, $"{inp_col}_day");
-            rv = rv.add_column(System.TypeCode.String, $"{inp_col}_month");
-            rv = rv.add_column(System.TypeCode.String, $"{inp_col}_hour");
-            rv = rv.add_column(System.TypeCode.String, $"{inp_col}_min");
+            rv = rv.add_column(System.TypeCode.Int32, $"{inp_col}_day_of_month");
+            rv = rv.add_column(System.TypeCode.Int32, $"{inp_col}_month");
+            rv = rv.add_column(System.TypeCode.Int32, $"{inp_col}_hour");
+            rv = rv.add_column(System.TypeCode.Int32, $"{inp_col}_min");
+            rv = rv.add_column(System.TypeCode.Int32, $"{inp_col}_seconds_of_day");
            return rv;
         };
         ViewStream rs = new MapStream(parent_stream,
@@ -23,9 +25,11 @@ namespace bts.SilverPine {
            MutableBlueRow rv = new MutableBlueRow(s, r);
            rv.setValue($"{inp_col}_date", dt.Date);
            rv.setValue($"{inp_col}_day", dt.DayOfWeek.ToString());
-            rv.setValue($"{inp_col}_month", dt.Date.Month.ToString());
+            rv.setValue($"{inp_col}_day_of_month", dt.Day);
+            rv.setValue($"{inp_col}_month", dt.Date.Month);
            rv.setValue($"{inp_col}_hour", dt.Hour);
            rv.setValue($"{inp_col}_min", dt.Minute);
+            rv.setValue($"{inp_col}_seconds_of_day", (dt - dt.Date).TotalSeconds);
            return rv;
         });
         mySetRealSource(rs);
@@ -39,7 +39,7 @@ namespace bts.SilverPine {
         mySetRealSource(rs);
      }

-      // for each row, does a row[new_ol_name] = row[col1] - rows[row_ffset][col2]
+      // for each row, does a row[new_col_name] = row[col1] - rows[row_ffset][col2]
      // we should generlize this for any math operation
      public ColumnDifferenceStream(ViewStream parent_stream, string sort_col_name, int row_offset, string col1, string col2, string diff_col_name, double optional_perc_diff_filter = 0.0) {
         // use GlobalMapStream
@@ -91,19 +91,32 @@ namespace bts.SilverPine {
         DataPacket dp = dp_obj as DataPacket;
         List<IBlueRow> rows = dp.newRows.to_list();
         var f1_rows = rows.Where(r => _f1(r));
-         Log.assert(f1_rows.Count() == 1);
+         //Log.assert(f1_rows.Count() == 1);
         var f2_rows = rows.Where(r => _f2(r));
-         Log.assert(f2_rows.Count() == 1);
-         IBlueRow f1_r = f1_rows.ElementAt(0);
-         IBlueRow f2_r = f2_rows.ElementAt(0);
-         double diff = f1_r.get<double>(_c1) - f2_r.get<double>(_c2);
-         IBlueTableSchema out_schema = f1_r.schema.add_columns(Tuple.Create(typeof(double), "diff"));
-         BlueTable out_table = new BlueTable(out_schema);
-         MutableBlueRow out_row = new MutableBlueRow(out_schema, f1_r);
-         out_row.setValue("diff", diff);
-         out_table.addRow(out_row);
-         DataPacket dp_out = BlueDataUtility.build_dp(out_table, myLastValue as DataPacket);
-         myFireEvent(dp_out);
+         //Log.assert(f2_rows.Count() == 1);
+         if (f1_rows != null && f1_rows.Count() > 0 && f2_rows != null && f2_rows.Count() > 0) {
+            IBlueRow f1_r = f1_rows.ElementAt(0);
+            IBlueRow f2_r = f2_rows.ElementAt(0);
+            double diff = f1_r.get<double>(_c1) - f2_r.get<double>(_c2);
+            IBlueTableSchema out_schema = f1_r.schema.add_columns(Tuple.Create(typeof(double), "diff"));
+            BlueTable out_table = new BlueTable(out_schema);
+            MutableBlueRow out_row = new MutableBlueRow(out_schema, f1_r);
+            out_row.setValue("diff", diff);
+            out_table.addRow(out_row);
+            DataPacket dp_out = BlueDataUtility.build_dp(out_table, myLastValue as DataPacket);
+            myFireEvent(dp_out);
+         } else {
+            IBlueRow f1_r = rows.ElementAt(0);
+            IBlueRow f2_r = rows.ElementAt(0);
+            double diff = Double.NaN;
+            IBlueTableSchema out_schema = f1_r.schema.add_columns(Tuple.Create(typeof(double), "diff"));
+            BlueTable out_table = new BlueTable(out_schema);
+            MutableBlueRow out_row = new MutableBlueRow(out_schema, f1_r);
+            out_row.setValue("diff", diff);
+            out_table.addRow(out_row);
+            DataPacket dp_out = BlueDataUtility.build_dp(out_table, myLastValue as DataPacket);
+            myFireEvent(dp_out);
+         }

      }
   }
@@ -0,0 +1,73 @@
+namespace Bts.DataStream;
+
+using BCG;
+
+using bts.api.core;
+
+using Bts.DataStreamCore;
+
+using System;
+using System.Collections;
+using System.Collections.Generic;
+
+
+public class TableUpdateDataPacket<T> : IDataPacket<T> where T : IBlueRow {
+   readonly TableUpdate<T, int> table;
+   readonly IBlueTableSchema schema;
+
+   public TableUpdateDataPacket(TableUpdate<T, int> table, IBlueTableSchema schema) {
+      this.table = table;
+      this.schema = schema;
+   }
+
+   public IBlueRowCollection<T> newRows {
+      get {
+         var rv = new BlueTableImmutable<T>(schema);
+         return rv;
+      }
+   }
+
+   public IBlueRowCollection<T> addedRows => throw new NotImplementedException();
+
+   public IBlueRowCollection<T> deletedRows => throw new NotImplementedException();
+
+   class MyRowCollection : IBlueRowCollection<T> {
+      readonly TableUpdate<T, int> table;
+      readonly IEnumerable<T> items;
+      Lazy<SortedList<int, T>> itemList;
+
+      public MyRowCollection(IEnumerable<T> items, IBlueTableSchema schema) {
+         this.items = items;
+         this.schema = schema;
+         itemList = new(() => {
+            var rv = new SortedList<int, T>();
+            foreach (var item in this.items) {
+               rv.Add(item.Id(), item);
+            }
+            return rv;
+         });
+      }
+
+      public int Count => itemList.Value.Count;
+      public bool IsSynchronized => false;
+      public object SyncRoot => this;
+
+      public T this[int id] => itemList.Value[id];
+
+      public IBlueTableSchema schema { get; }
+
+      public bool containsID(int id) =>
+         itemList.Value.ContainsKey(id);
+
+      public void CopyTo(Array array, int index) {
+      }
+
+      public IEnumerator<T> GetEnumerator() {
+         throw new NotImplementedException();
+      }
+
+      IEnumerator IEnumerable.GetEnumerator() {
+         throw new NotImplementedException();
+      }
+   }
+}
@@ -0,0 +1,127 @@
+namespace Bts.DataStreamCore;
+
+using BCG;
+
+using bts.utils;
+
+using mbts.utils;
+
+using Spark.Core;
+
+using System;
+
+using System.Collections;
+using System.Collections.Generic;
+using System.Linq;
+
+public class BlueColumnTable : IBlueRowCollection {
+   readonly Variant[][] columns;
+   readonly List<int> rowIds;
+   readonly int idColIndex;
+
+   public BlueColumnTable(IBlueTableSchema schema, int capacity = 4) {
+      ExceptionAssert.Argument.GreaterThanEqualTo(capacity, 0);
+      rowIds = new(capacity);
+      columns = new Variant[schema.columnInfo.Length][];
+      if (columns.Length == 0) {
+         throw new ArgumentException("Schema contains no columns", nameof(schema));
+      }
+      if (!schema.columnInfo.Select(c => c.colName).TryFindIndex("id", out idColIndex)) {
+         throw new ArgumentException("Schema contains no 'id' column", nameof(schema));
+      }
+      this.schema = schema;
+      for (int i = 0; i < columns.Length; i++) {
+         columns[schema.columnInfo[i].colNo] = new Variant[capacity];
+      }
+   }
+
+   public int Count => rowIds.Count;
+   public IBlueTableSchema schema { get; }
+
+   public IBlueRow this[int id] {
+      get {
+         var idx = rowIds.BinarySearch(id);
+         if (idx < 0) {
+            throw new KeyNotFoundException($"No row with id {id}");
+         }
+         return new MyRow(this, idx);
+      }
+   }
+
+   public void Add(Variant[] values) {
+      var id = values[idColIndex].VInt32;
+      var idx = rowIds.BinarySearch(id);
+      if (idx >= 0) {
+         throw new ArgumentException($"Row with id {id} already exists.", nameof(values));
+      }
+      idx = ~idx;
+      for (int i = 0; i < columns.Length; i++) {
+         int len = rowIds.Count;
+         ArrayHelper.Insert(ref columns[i], idx, values[i], ref len);
+      }
+      rowIds.Insert(idx, id);
+   }
+
+   public void AddOrUpdate(Variant[] values) {
+      var id = values[idColIndex].VInt32;
+      var idx = rowIds.BinarySearch(id);
+      if (idx >= 0) {
+         rowIds[idx] = id;
+         for (int i = 0; i < columns.Length; i++) {
+            columns[idx][i] = values[i];
+         }
+         return;
+      }
+      idx = ~idx;
+      int len;
+      for (int i = 0; i < columns.Length; i++) {
+         len = rowIds.Count;
+         ArrayHelper.Insert(ref columns[i], idx, values[i], ref len);
+      }
+      rowIds.Insert(idx, id);
+   }
+
+   public void Clear() =>
+      rowIds.Clear();
+
+   public bool Remove(int id) {
+      var idx = rowIds.BinarySearch(id);
+      if (idx < 0) {
+         return false;
+      }
+      rowIds.RemoveAt(idx);
+      for (int i = 0; i < columns.Length; i++) {
+         ArrayHelper.remove_at(ref columns[i], idx);
+      }
+      return true;
+   }
+
+   public void Trim() {
+      for (int i = 0; i < columns.Length; i++) {
+         ArrayHelper.trim(ref columns[i], rowIds.Count);
+      }
+   }
+
+   public bool containsID(int id) =>
+      rowIds.BinarySearch(id) >= 0;
+
+   public IEnumerator GetEnumerator() {
+      var count = rowIds.Count;
+      for (int i = 0; i < count; i++) {
+         yield return new MyRow(this, i);
+      }
+   }
+
+   class MyRow : IBlueRow {
+      readonly BlueColumnTable table;
+      readonly int rowIdx;
+
+      public MyRow(BlueColumnTable table, int rowIdx) {
+         this.table = table;
+         this.rowIdx = rowIdx;
+      }
+
+      public object this[int colNo] => table.columns[colNo][rowIdx].GetValue();
+      public IBlueTableSchema schema => table.schema;
+   }
+}
@@ -18,6 +18,9 @@ using System.Text.Json.Nodes;
 using Newtonsoft.Json;
 using BCG;
 using System.Runtime.CompilerServices;
+using bts;
+using Spark.Core;
+using System.Numerics;

 namespace bts {

@@ -89,6 +92,132 @@ namespace BCG {
      public static DateTime get_datetime(this IBlueRow r, int col_no, DateTime err_value = default(DateTime)) {
         return BlueDataUtility.get_row_datetime_value(r, col_no, err_value);
      }
+
+      public static double? GetFloat64(this IBlueRow r, int colNo) =>
+         GetFloat64(r[colNo]);
+
+      public static double? GetFloat64(this IBlueRow r, string colName) =>
+         GetFloat64(r[colName]);
+
+      public static double? GetFloat64(object obj) {
+         if (obj is double d) {
+            return d;
+         }
+         if (obj is int i32) {
+            return i32;
+         }
+         if (obj is long i64) {
+            return i64;
+         }
+         if (obj is string s) {
+            return double.TryParse(s, out d) ? d : null;
+         }
+         if (obj is uint u32) {
+            return u32;
+         }
+         if (obj is ulong u64) {
+            return u64;
+         }
+         if (obj is short i16) {
+            return i16;
+         }
+         if (obj is ushort u16) {
+            return u16;
+         }
+         return null;
+      }
+
+      public static int? GetInt32(this IBlueRow r, int colNo) =>
+         GetInt32(r[colNo]);
+
+      public static int? GetInt32(this IBlueRow r, string colName) =>
+         GetInt32(r[colName]);
+
+      public static int? GetInt32(object obj) {
+         checked {
+            if (obj is double d) {
+               return (int)d;
+            }
+            if (obj is int i32) {
+               return i32;
+            }
+            if (obj is long i64) {
+               return (int)i64;
+            }
+            if (obj is string s) {
+               return double.TryParse(s, out d) ? (int)d : null;
+            }
+            if (obj is uint u32) {
+               return (int)u32;
+            }
+            if (obj is ulong u64) {
+               return (int)u64;
+            }
+            if (obj is short i16) {
+               return i16;
+            }
+            if (obj is ushort u16) {
+               return u16;
+            }
+         }
+         return null;
+      }
+
+      public static long? GetInt64(this IBlueRow r, int colNo) =>
+         GetInt64(r[colNo]);
+
+      public static long? GetInt64(this IBlueRow r, string colName) =>
+         GetInt64(r[colName]);
+
+      public static long? GetInt64(object obj) {
+         checked {
+            if (obj is double d) {
+               return (long)d;
+            }
+            if (obj is int i32) {
+               return i32;
+            }
+            if (obj is long i64) {
+               return i64;
+            }
+            if (obj is string s) {
+               return double.TryParse(s, out d) ? (long)d : null;
+            }
+            if (obj is uint u32) {
+               return u32;
+            }
+            if (obj is ulong u64) {
+               return (long)u64;
+            }
+            if (obj is short i16) {
+               return i16;
+            }
+            if (obj is ushort u16) {
+               return u16;
+            }
+         }
+         return null;
+      }
+
+      public static DateTime? GetUnixEpochDateTime(this IBlueRow row, string colName) =>
+         GetUnixEpochDateTime(row[colName]);
+
+      public static DateTime? GetUnixEpochDateTime(this IBlueRow row, int colNo) =>
+         GetUnixEpochDateTime(row[colNo]);
+
+      public static DateTime? GetUnixEpochDateTime(object obj) {
+         var tsNanos = GetInt64(obj);
+         return tsNanos.HasValue ? DateTime.UnixEpoch.Add(new TimeSpan(tsNanos.Value / 100)) : null;
+      }
+
+      public static DateTime? GetDateTime(object obj) {
+         var res = _get_datetime_value(obj, DateTime.MinValue);
+         if (res == DateTime.MinValue) {
+            return null;
+         }
+         return res;
+      }
+
      public static T get_enum<T>(this IBlueRow r, string col_name, T err_value = default(T)) where T : struct {
         return BlueDataUtility.get_row_enum_value<T>(r, col_name, err_value);
      }
@@ -207,6 +336,10 @@ namespace BCG {
         return rv;
      }

+      public static bool contains_column(this IBlueTableSchema input_schema, string col) {
+         return input_schema.columnNames.Contains(col);
+      }
+
      public static IBlueTableSchema add_columns(this IBlueTableSchema input_schema, params Tuple<Type, string>[] cols) {
         List<ColumnInfo> cis = new List<ColumnInfo>(input_schema.columnInfo);
         foreach (Tuple<Type, string> c in cols)
@@ -1719,8 +1852,10 @@ namespace BCG {

      public static int[] get_column_indices(IBlueTableSchema schema, string[] col_names) {
         List<int> rv = new List<int>();
-         foreach (string c in col_names)
+         foreach (string c in col_names) {
+            Log.assert(schema.contains_column(c), $"Could not find col: {c} in table: {schema.tableName}");
            rv.Add(schema[c].colNo);
+         }
         return rv.ToArray();
      }

@@ -2207,23 +2342,57 @@ namespace BCG {
         try {
            if (v == null || v == DBNull.Value)
               return error_val;
-            else {
-               if (v is DateTime)
-                  return (DateTime)v;
-               else if (v is string) {
-                  DateTime rv_dt = error_val;
-                  DateTime.TryParse((string)v, out rv_dt);
+
+            if (v is DateTime) {
+               return (DateTime)v;
+            } else if (v is string) {
+               DateTime rv_dt;
+
+               // Try parsing the string as a DateTime
+               if (DateTime.TryParse((string)v,
+                                     System.Globalization.CultureInfo.InvariantCulture,
+                                     System.Globalization.DateTimeStyles.RoundtripKind,
+                                     out rv_dt)) {
                  return rv_dt;
               } else {
-                  double oa_date = (double)Convert.ChangeType(v, typeof(Double));
-                  return DateTime.FromOADate(oa_date);
+                  return error_val; // Return the error value if parsing fails
               }
+            } else {
+               // Assume the input can be converted to a double and use FromOADate
+               double oa_date = (double)Convert.ChangeType(v, typeof(Double));
+               return DateTime.FromOADate(oa_date);
            }
         } catch {
            return error_val;
         }
      }

+      public static TimeSpan? get_timespan(this IBlueRow r, int col_no) =>
+         get_timespan(r[col_no]);
+
+      public static TimeSpan? get_timespan(this IBlueRow r, string col_name) =>
+         get_timespan(r[col_name]);
+
+      public static TimeSpan? get_timespan(object obj) {
+         if (obj is TimeSpan ts) {
+            return ts;
+         }
+         var str = obj as string ?? obj?.ToString();
+         if (str is null) {
+            return null;
+         }
+         if (TimeSpan.TryParse(str, out ts)) {
+            return ts;
+         }
+         if (str.StartsWith("PT")) {
+            try {
+               // Parse ISO 8601 duration
+               return System.Xml.XmlConvert.ToTimeSpan(str);
+            } catch {
+            }
+         }
+         return null;
+      }
      #endregion

      #region CSV methods
@@ -0,0 +1,56 @@
+#nullable enable
+namespace Bts.DataStreamCore;
+
+using BCG;
+
+using System;
+using System.Collections;
+using System.Collections.Generic;
+using System.Linq;
+
+public class BlueRowCollection : IBlueRowCollection {
+   readonly ICollection source;
+   readonly Func<int, IBlueRow> findRow;
+   readonly Func<int, bool> containsKey;
+
+   public BlueRowCollection(ICollection source, IBlueTableSchema schema, Func<int, IBlueRow>? findRow = null, Func<int, bool>? containsKey = null) {
+      this.source = source;
+      this.schema = schema;
+      if (findRow is null || containsKey is null) {
+         if (source is IBlueRowCollection brc) {
+            findRow ??= id => brc[id];
+            containsKey ??= brc.containsID;
+         } else if (source is IDictionary dict) {
+            findRow ??= id => dict[id] as IBlueRow ?? throw new KeyNotFoundException();
+            containsKey ??= id => dict.Contains(id);
+         } else if (source is IDictionary<int, IBlueRow> genDict) {
+            findRow ??= id => genDict[id];
+            containsKey ??= genDict.ContainsKey;
+         } else if (source is IReadOnlyDictionary<int, IBlueRow> roGenDict) {
+            findRow ??= id => roGenDict[id];
+            containsKey = roGenDict.ContainsKey;
+         } else {
+            var cachedRows = new Lazy<Dictionary<int, IBlueRow>>(() =>
+               source.Cast<IBlueRow>().ToDictionary(r => r.Id()));
+            findRow ??= id => cachedRows.Value[id];
+            // Don't remove this lambda expression. Intellisense may suggest it, but that would
+            // instantiate 'cachedRows' immediately and defeat the purpose of using Lazy<T>.
+            containsKey ??= id => cachedRows.Value.ContainsKey(id);
+         }
+      }
+      this.findRow = findRow;
+      this.containsKey = containsKey;
+   }
+
+   public IBlueRow this[int id] => findRow(id);
+
+   public IBlueTableSchema schema { get; }
+
+   public int Count => source.Count;
+
+   public bool containsID(int id) =>
+      containsKey(id);
+
+   public IEnumerator GetEnumerator() =>
+      source.GetEnumerator();
+}
@@ -0,0 +1,64 @@
+#nullable enable
+namespace Bts.DataStreamCore;
+
+using BCG;
+
+using bts.collections;
+using bts.utils;
+
+using System;
+using System.Collections.Generic;
+
+
+public class BlueRowColumnMask : IBlueRow {
+   readonly IBlueRow row;
+   readonly int[] hiddenColumns;
+
+   public BlueRowColumnMask(IBlueRow row, params int[] hiddenColumns) {
+      this.row = row;
+      this.hiddenColumns = hiddenColumns;
+      Array.Sort(hiddenColumns);
+   }
+
+   /// <summary>
+   /// Provides a way to explicitly skip sorting hiddenColumns. This is done at the callers risk,
+   /// as the index (int) operator performs a binary search on hiddenColumns.
+   /// </summary>
+   /// <param name="row">The underlying row whose columns to hide</param>
+   /// <param name="sortHiddenColumns">Whether or not to sort <paramref name="hiddenColumns"/> in
+   /// the constructor. They are sorted by default to ensure correct behavior in the index (int)
+   /// operator, which performs a binary search on this array.</param>
+   /// <param name="hiddenColumns">The columns (indices) to hide</param>
+   public BlueRowColumnMask(IBlueRow row, bool sortHiddenColumns, params int[] hiddenColumns) {
+      this.row = row;
+      this.hiddenColumns = hiddenColumns;
+      if (sortHiddenColumns) {
+         Array.Sort(hiddenColumns);
+      }
+   }
+
+   public object? this[string colName] {
+      get {
+         for (int i = 0; i < hiddenColumns.Length; i++) {
+            if (row.schema.columnInfo[i].colName == colName) {
+               throw new KeyNotFoundException();
+            }
+         }
+         return row[colName];
+      }
+   }
+   public object? this[int index] {
+      get {
+         if (index > row.schema.columnInfo.Length - hiddenColumns.Length) {
+            throw new IndexOutOfRangeException();
+         }
+         int pos = hiddenColumns.BinarySearch(index);
+         if (pos < 0) {
+            pos = ~pos;
+         }
+         return row[index + pos];
+      }
+   }
+
+   public IBlueTableSchema schema => row.schema;
+}
@@ -7,7 +7,7 @@ namespace BCG
   /// Used for enumerating an IEnumerable (ex ICollection). Furthermore, this is the 
   /// default Enueramtor used when enumerating asn IBlueRow
   /// </summary>
-   public class BlueRowEnumerator : IEnumerator
+   public class BlueRowEnumerator : IDictionaryEnumerator
      {
         protected int myPos = -1;
         protected int myRowSize = -1;
@@ -18,6 +18,11 @@ namespace BCG
            myRow = row;
            myRowSize = row.schema.columnNames.Length;
         }
+
+         public DictionaryEntry Entry =>
+            new(myRow.schema.columnInfo[myPos].colName, myRow[myPos]);
+         public object Key => myRow.schema.columnInfo[myPos].colName;
+         public object Value => myRow[myPos];
         
         #region IEnumerator members
         public void Reset()  {myPos = 0;}
@@ -0,0 +1,30 @@
+#nullable enable
+namespace BCG;
+
+using System;
+using System.Collections;
+
+public class BlueRowKeyCollection : ICollection {
+   readonly ColumnInfo[] columns;
+
+   public BlueRowKeyCollection(ColumnInfo[] columns) =>
+      this.columns = columns;
+
+   public int Count => columns.Length;
+
+   public bool IsSynchronized => false;
+
+   public object SyncRoot => this;
+
+   public void CopyTo(Array array, int index) {
+      for (int i = 0; i < columns.Length; i++) {
+         array.SetValue(columns[i].colName, i + index);
+      }
+   }
+
+   public IEnumerator GetEnumerator() {
+      for (int i = 0; i < columns.Length; i++) {
+         yield return columns[i].colName;
+      }
+   }
+}
@@ -0,0 +1,44 @@
+#nullable enable
+namespace Bts.DataStreamCore;
+
+using BCG;
+
+using System.Linq;
+using System.Reflection;
+
+public static class BlueRowReflectionFactory {
+   public static IBlueRow Create<T>(T row, string? tableName = null) =>
+      new BlueRowReflection<T>(row);
+
+   class BlueRowReflection<T> : IBlueRow {
+      public static readonly ColumnInfo[] ColumnInfos;
+      public static readonly PropertyInfo[] ColumnProps;
+
+      readonly T row;
+      readonly string? tableName;
+
+      static BlueRowReflection() {
+         var colTups = typeof(T).GetProperties(BindingFlags.Instance)
+            .Select(p => (Prop: p, Attr: p.GetCustomAttribute<BlueTableColumnAttribute>()))
+            .Where(p => p.Attr is not null)
+            .ToList();
+         ColumnInfos = new ColumnInfo[colTups.Count];
+         ColumnProps = new PropertyInfo[colTups.Count];
+         for (int i = 0; i < colTups.Count; i++) {
+            var (prop, attr) = colTups[i];
+            ColumnInfos[i] = new(attr?.Name ?? prop.Name, i, prop.PropertyType);
+            ColumnProps[i] = prop;
+         }
+      }
+
+      public BlueRowReflection(T row, string? tableName = null) {
+         this.row = row;
+         this.tableName = tableName;
+      }
+
+      public object? this[int index] => ColumnProps[index].GetValue(row);
+
+      public IBlueTableSchema schema =>
+         new BlueTableSchema(ColumnInfos, tableName ?? typeof(T).Name);
+   }
+}
@@ -0,0 +1,311 @@
+#nullable enable
+namespace Bts.DataStreamCore;
+
+using BCG;
+
+using mbts.utils;
+
+using System;
+using System.Collections;
+using System.Collections.Generic;
+using System.Diagnostics.CodeAnalysis;
+using System.Linq;
+
+public class BlueRowStore<T1, T2, T3, T4, T5, T6> : IBlueRow, IReadOnlyDictionary<string, object?> {
+   readonly string? name;
+   IBlueTableSchema? _schema;
+
+   public BlueRowStore() { }
+
+   public BlueRowStore(
+      int id,
+      (string Name, T1 Value) item1,
+      (string Name, T2 Value) item2,
+      (string Name, T3 Value) item3,
+      (string Name, T4 Value) item4,
+      (string Name, T5 Value) item5,
+      (string Name, T6 Value) item6
+   ) {
+      Id = id;
+      Item1 = item1;
+      Item2 = item2;
+      Item3 = item3;
+      Item4 = item4;
+      Item5 = item5;
+      Item6 = item6;
+   }
+
+   public int Id { get; set; }
+   public (string Name, T1 Value) Item1 { get; set; }
+   public (string Name, T2 Value) Item2 { get; set; }
+   public (string Name, T3 Value) Item3 { get; set; }
+   public (string Name, T4 Value) Item4 { get; set; }
+   public (string Name, T5 Value) Item5 { get; set; }
+   public (string Name, T6 Value) Item6 { get; set; }
+
+   public int Count => 5;
+   public IReadOnlyList<string> Keys => new KeyList(this);
+   public IReadOnlyList<object?> Values => new ValueList(this);
+
+   public IBlueTableSchema schema =>
+      _schema ??= new BlueTableSchema(name, new ColumnInfo[]{
+         new("id", 0, typeof(int)),
+         new(Item1.Name, 1, typeof(T1)),
+         new(Item2.Name, 2, typeof(T2)),
+         new(Item3.Name, 3, typeof(T3)),
+         new(Item4.Name, 4, typeof(T4)),
+         new(Item5.Name, 5, typeof(T5)),
+         new(Item6.Name, 6, typeof(T6)),
+      });
+
+   ICollection IDictionary.Keys => new KeyList(this);
+   ICollection IDictionary.Values => new ValueList(this);
+
+   IEnumerable<string> IReadOnlyDictionary<string, object?>.Keys => Keys;
+   IEnumerable<object?> IReadOnlyDictionary<string, object?>.Values => Values;
+
+   bool ICollection.IsSynchronized => false;
+   object ICollection.SyncRoot => this;
+
+   public object? this[string colName] {
+      get {
+         if (colName == "id") {
+            return Id;
+         }
+         if (colName == Item1.Name) {
+            return Item1.Value;
+         }
+         if (colName == Item2.Name) {
+            return Item2.Value;
+         }
+         if (colName == Item3.Name) {
+            return Item3.Value;
+         }
+         if (colName == Item4.Name) {
+            return Item4.Value;
+         }
+         if (colName == Item5.Name) {
+            return Item5.Value;
+         }
+         if (colName == Item6.Name) {
+            return Item6.Value;
+         }
+         throw new KeyNotFoundException();
+      }
+      set {
+         if (colName == "id") {
+            Id = ExceptionAssert.Cast(value).To<int>();
+         }
+         if (colName == Item1.Name) {
+            Item1 = (Item1.Name, ExceptionAssert.Cast(value).To<T1>());
+         }
+         if (colName == Item2.Name) {
+            Item2 = (Item2.Name, ExceptionAssert.Cast(value).To<T2>());
+         }
+         if (colName == Item3.Name) {
+            Item3 = (Item3.Name, ExceptionAssert.Cast(value).To<T3>());
+         }
+         if (colName == Item4.Name) {
+            Item4 = (Item4.Name, ExceptionAssert.Cast(value).To<T4>());
+         }
+         if (colName == Item5.Name) {
+            Item5 = (Item5.Name, ExceptionAssert.Cast(value).To<T5>());
+         }
+         if (colName == Item6.Name) {
+            Item6 = (Item6.Name, ExceptionAssert.Cast(value).To<T6>());
+         }
+         throw new KeyNotFoundException();
+      }
+   }
+
+   public object? this[int index] => index switch {
+      0 => Id,
+      1 => Item1.Value,
+      2 => Item2.Value,
+      3 => Item3.Value,
+      4 => Item4.Value,
+      5 => Item5.Value,
+      6 => Item6.Value,
+      _ => throw new IndexOutOfRangeException()
+   };
+
+   object? IDictionary.this[object key] {
+      get => this[ExceptionAssert.Cast(key).To<string>()];
+      set => this[ExceptionAssert.Cast(key).To<string>()] = value;
+   }
+
+   public void CopyTo(Array array, int index) {
+      array.SetValue(Id, index++);
+      array.SetValue(Item1.Value, index++);
+      array.SetValue(Item2.Value, index++);
+      array.SetValue(Item3.Value, index++);
+      array.SetValue(Item4.Value, index++);
+      array.SetValue(Item5.Value, index++);
+      array.SetValue(Item6.Value, index++);
+   }
+
+   public bool Contains(object key) =>
+      key is string keystr && ContainsKey(keystr);
+
+   public bool ContainsKey(string key) =>
+      Keys.Contains(key);
+
+   public void SetId(int id) =>
+      Id = id;
+
+   public void SetValue1(T1 value) =>
+      Item1 = (Item1.Name, value);
+
+   public void SetValue1(string colName, T1 value) =>
+      Item1 = (colName, value);
+
+   public void SetValue2(T2 value) =>
+      Item2 = (Item2.Name, value);
+
+   public void SetValue2(string colName, T2 value) =>
+      Item2 = (colName, value);
+
+   public void SetValue3(T3 value) =>
+      Item3 = (Item3.Name, value);
+
+   public void SetValue3(string colName, T3 value) =>
+      Item3 = (colName, value);
+
+   public void SetValue4(T4 value) =>
+      Item4 = (Item4.Name, value);
+
+   public void SetValue4(string colName, T4 value) =>
+      Item4 = (colName, value);
+
+   public void SetValue5(T5 value) =>
+      Item5 = (Item5.Name, value);
+
+   public void SetValue5(string colName, T5 value) =>
+      Item5 = (colName, value);
+
+   public void SetValue6(T6 value) =>
+      Item6 = (Item6.Name, value);
+
+   public void SetValue6(string colName, T6 value) =>
+      Item6 = (colName, value);
+
+   public bool TryGetValue(string key, [MaybeNullWhen(false)] out object? value) {
+      if (!Keys.TryFindIndex(key, out int index)) {
+         value = null;
+         return false;
+      }
+      value = this[index];
+      return true;
+   }
+
+   void IDictionary.Add(object key, object? value) =>
+      throw new NotSupportedException();
+
+   void IDictionary.Clear() =>
+      throw new NotSupportedException();
+
+   void IDictionary.Remove(object key) =>
+      throw new NotSupportedException();
+
+   IDictionaryEnumerator IDictionary.GetEnumerator() =>
+      throw new NotImplementedException();
+
+   IEnumerator IEnumerable.GetEnumerator() =>
+      throw new NotImplementedException();
+
+   public IEnumerator<KeyValuePair<string, object?>> GetEnumerator() {
+      throw new NotImplementedException();
+   }
+
+   class KeyList : ICollection, IReadOnlyList<string> {
+      readonly BlueRowStore<T1, T2, T3, T4, T5, T6> store;
+
+      public KeyList(BlueRowStore<T1, T2, T3, T4, T5, T6> store) =>
+         this.store = store;
+
+      public int Count => 5;
+      bool ICollection.IsSynchronized => false;
+      object ICollection.SyncRoot => this;
+
+      public string this[int index] =>
+         index switch {
+            0 => "id",
+            1 => store.Item1.Name,
+            2 => store.Item2.Name,
+            3 => store.Item3.Name,
+            4 => store.Item4.Name,
+            5 => store.Item5.Name,
+            6 => store.Item6.Name,
+            _ => throw new IndexOutOfRangeException()
+         };
+
+      public void CopyTo(Array array, int index) {
+         array.SetValue(store.Id, index++);
+         array.SetValue(store.Item1.Value, index++);
+         array.SetValue(store.Item2.Value, index++);
+         array.SetValue(store.Item3.Value, index++);
+         array.SetValue(store.Item4.Value, index++);
+         array.SetValue(store.Item5.Value, index++);
+         array.SetValue(store.Item6.Value, index++);
+      }
+
+      public IEnumerator<string> GetEnumerator() {
+         yield return "id";
+         yield return store.Item1.Name;
+         yield return store.Item2.Name;
+         yield return store.Item3.Name;
+         yield return store.Item4.Name;
+         yield return store.Item5.Name;
+         yield return store.Item6.Name;
+      }
+
+      IEnumerator IEnumerable.GetEnumerator() =>
+         GetEnumerator();
+   }
+
+   class ValueList : ICollection, IReadOnlyList<object?> {
+      readonly BlueRowStore<T1, T2, T3, T4, T5, T6> store;
+
+      public ValueList(BlueRowStore<T1, T2, T3, T4, T5, T6> store) =>
+         this.store = store;
+
+      public object? this[int index] =>
+         index switch {
+            0 => store.Id,
+            1 => store.Item1.Value,
+            2 => store.Item2.Value,
+            3 => store.Item3.Value,
+            4 => store.Item4.Value,
+            5 => store.Item5.Value,
+            6 => store.Item6.Value,
+            _ => throw new IndexOutOfRangeException()
+         };
+
+      public int Count => 5;
+      public bool IsSynchronized => false;
+      public object SyncRoot => this;
+
+      public void CopyTo(Array array, int index) {
+         array.SetValue(store.Id, index++);
+         array.SetValue(store.Item1.Value, index++);
+         array.SetValue(store.Item2.Value, index++);
+         array.SetValue(store.Item3.Value, index++);
+         array.SetValue(store.Item4.Value, index++);
+         array.SetValue(store.Item5.Value, index++);
+         array.SetValue(store.Item6.Value, index++);
+      }
+
+      public IEnumerator<object?> GetEnumerator() {
+         yield return store.Id;
+         yield return store.Item1.Value;
+         yield return store.Item2.Value;
+         yield return store.Item3.Value;
+         yield return store.Item4.Value;
+         yield return store.Item5.Value;
+         yield return store.Item6.Value;
+      }
+
+      IEnumerator IEnumerable.GetEnumerator() =>
+         GetEnumerator();
+   }
+}
@@ -0,0 +1,32 @@
+#nullable enable
+namespace BCG;
+
+using System;
+using System.Collections;
+
+public class BlueRowValueCollection : ICollection {
+   readonly IBlueRow row;
+
+   public BlueRowValueCollection(IBlueRow row) =>
+      this.row = row;
+
+   public int Count => row.schema.columnInfo.Length;
+
+   public bool IsSynchronized => false;
+
+   public object SyncRoot => this;
+
+   public void CopyTo(Array array, int index) {
+      var length = row.schema.columnInfo.Length;
+      for (int i = 0; i < length; i++) {
+         array.SetValue(row[i], i + index);
+      }
+   }
+
+   public IEnumerator GetEnumerator() {
+      var length = row.schema.columnInfo.Length;
+      for (int i = 0; i < length; i++) {
+         yield return row[i];
+      }
+   }
+}
@@ -0,0 +1,17 @@
+#nullable enable
+namespace Bts.DataStreamCore;
+
+using System;
+
+[AttributeUsage(AttributeTargets.Property)]
+public class BlueTableColumnAttribute : Attribute {
+   readonly int index;
+
+   public BlueTableColumnAttribute(string? name = null, int index = -1) {
+      Name = name;
+      this.index = index;
+   }
+
+   public string? Name { get; }
+   public int? Index => index >= 0 ? index : null;
+}
@@ -196,6 +196,24 @@ public class BlueTableImmutable<T> : IBlueRowCollection<T> where T : IBlueRow {
         return new DataPacket<T>(newRows, addedRows, removedRows);
      }

+      public IDataPacket<T> ToDataPacketT(
+         out BlueTableImmutable<T> newRows,
+         out BlueTableImmutable<T> addedRows,
+         out BlueTableImmutable<T> removedRows,
+         bool reset = true
+      ) {
+         table.RemoveRange(removed.Keys);
+         table.AddRange(added);
+         newRows = new BlueTableImmutable<T>(schema, table.ToImmutable());
+         addedRows = new BlueTableImmutable<T>(schema, added.ToImmutable());
+         removedRows = new BlueTableImmutable<T>(schema, removed.ToImmutable());
+         if (reset) {
+            added.Clear();
+            removed.Clear();
+         }
+         return new DataPacket<T>(newRows, addedRows, removedRows);
+      }
+
      public BlueTableImmutable<T> ToImmutable() =>
         new(schema, table.ToImmutable());
   }
@@ -0,0 +1,12 @@
+#nullable enable
+namespace Bts.DataStreamCore;
+
+using System;
+
+[AttributeUsage(AttributeTargets.Class)]
+public class BlueTableNameAttribute : Attribute {
+   public BlueTableNameAttribute(string name) =>
+      Name = name;
+
+   public string Name { get; }
+}
@@ -0,0 +1,40 @@
+#nullable enable
+namespace Bts.DataStreamCore;
+
+using BCG;
+
+using System;
+using System.Collections;
+using System.Collections.Generic;
+
+public class BlueTableSchemaBuilder : IEnumerable<(string, Type)> {
+   readonly string? tableName;
+   readonly List<(string Name, Type Type)> colDefs = new();
+
+   public BlueTableSchemaBuilder(string? tableName = null) =>
+      this.tableName = tableName;
+
+   public void Add((string Name, Type Type) tup) =>
+      Add(tup.Name, tup.Type);
+
+   public void Add<T>(string Name) =>
+      colDefs.Add((Name, typeof(T)));
+
+   public void Add(string Name, Type Type) =>
+      colDefs.Add((Name, Type));
+
+   public IEnumerator<(string, Type)> GetEnumerator() {
+      throw new NotImplementedException();
+   }
+
+   public IBlueTableSchema ToSchema(string? tableName = null) {
+      var colInfos = new ColumnInfo[colDefs.Count];
+      for (int i = 0; i < colInfos.Length; i++) {
+         colInfos[i] = new(colDefs[i].Name, i, colDefs[i].Type);
+      }
+      return new BlueTableSchema(colInfos, tableName ?? this.tableName);
+   }
+
+   IEnumerator IEnumerable.GetEnumerator() =>
+      colDefs.GetEnumerator();
+}
@@ -0,0 +1,16 @@
+#nullable enable
+namespace Bts.DataStreamCore;
+
+using BCG;
+
+using System;
+
+public class BlueTableSchemaFactory {
+   public static IBlueTableSchema Create(string tableName, params (string Name, Type Type)[] columns) {
+      var colInfos = new ColumnInfo[columns.Length];
+      for (int i = 0; i < columns.Length; i++) {
+         colInfos[i] = new(columns[i].Name, i, columns[i].Type);
+      }
+      return new BlueTableSchema(colInfos, tableName);
+   }
+}
@@ -0,0 +1,39 @@
+#nullable enable
+namespace Bts.DataStreamCore;
+
+using BCG;
+
+using System;
+using System.Collections.Generic;
+
+public static class BlueTableSelectorWrapper {
+   public static BlueTableSelectorWrapper<T> Create<T>(
+      IBlueRowCollection source, Func<IBlueRow, T> selector, IBlueTableSchema? schema = null) where T : IBlueRow =>
+      new(source, selector, schema);
+}
+
+public class BlueTableSelectorWrapper<T> : IBlueRowCollection<T> where T : IBlueRow {
+   readonly IBlueRowCollection source;
+   readonly Func<IBlueRow, T> selector;
+
+   public BlueTableSelectorWrapper(IBlueRowCollection source, Func<IBlueRow, T> selector, IBlueTableSchema? schema = null) {
+      this.source = source;
+      this.selector = selector;
+      this.schema = schema ?? source.schema;
+   }
+
+   public T this[int id] => selector(source[id]);
+
+   public IBlueTableSchema schema { get; }
+
+   public int Count => source.Count;
+
+   public bool containsID(int id) =>
+      source.containsID(id);
+
+   public IEnumerator<T> GetEnumerator() {
+      foreach (IBlueRow r in source) {
+         yield return selector(r);
+      }
+   }
+}
@@ -0,0 +1,64 @@
+namespace Bts.DataStreamCore;
+
+using BCG;
+
+using System;
+
+public struct ColumnIndex {
+   public readonly string[] ColumnNames;
+   public readonly int[] ColumnIndices;
+
+   public ColumnIndex(params string[] columnNames) {
+      ColumnNames = columnNames;
+      ColumnIndices = new int[columnNames.Length];
+   }
+
+   public readonly int this[int nameIndex] => ColumnIndices[nameIndex];
+
+   public static int Resolve(ColumnInfo[] columns, string name) {
+      var indices = Resolve(columns, new[] { name });
+      return indices[0];
+   }
+
+   public static (int, int) Resolve(ColumnInfo[] columns, string name1, string name2) {
+      var indices = Resolve(columns, new[] { name1, name2 });
+      return (indices[0], indices[1]);
+   }
+
+   public static int[] Resolve(ColumnInfo[] columns, params string[] names) {
+      var indices = new int[names.Length];
+      int nResolved = 0;
+      for (int i = 0; i < names.Length; i++) {
+         int origNResolved = nResolved;
+         for (int j = 0; j < columns.Length; j++) {
+            if (columns[j].colName == names[i]) {
+               indices[i] = j;
+               nResolved++;
+               break;
+            }
+         }
+         if (nResolved == origNResolved) {
+            throw new ArgumentException($"No column named '{names[i]}'", nameof(columns));
+         }
+      }
+      return indices;
+   }
+
+   public int Resolve(ColumnInfo[] columns, bool requireSuccess = true) {
+      int nResolved = 0;
+      for (int i = 0; i < ColumnNames.Length; i++) {
+         int origNResolved = nResolved;
+         for (int j = 0; j < columns.Length; j++) {
+            if (columns[j].colName == ColumnNames[i]) {
+               ColumnIndices[i] = j;
+               nResolved++;
+               break;
+            }
+         }
+         if (requireSuccess && nResolved == origNResolved) {
+            throw new ArgumentException($"No column named '{ColumnNames[i]}'", nameof(columns));
+         }
+      }
+      return nResolved;
+   }
+}
@@ -1,6 +1,8 @@
 #nullable enable
 using System;
 using System.Collections;
+using System.Collections.Generic;
+using System.Linq;

 namespace BCG
 {
@@ -23,11 +25,50 @@ namespace BCG
      /// half a million type checkings
      /// </summary>
      object? this[int index] {get;}
-      object? this[string colName] {get;}
+      object? this[string colName] {
+         get {
+            var cols = schema.columnInfo;
+            for (int i = 0; i < cols.Length; i++) {
+               if (cols[i].colName == colName) {
+                  return this[i];
+               }
+            }
+            throw new KeyNotFoundException();
+         }
+      }
+
+      object? IDictionary.this[object key] {
+         get => key is string s
+            ? this[s]
+            : throw new ArgumentException($"Key must be of type string (actually {key?.GetType()})", nameof(key));
+         set => throw new NotImplementedException();
+      }
+
+      ICollection IDictionary.Keys => new BlueRowKeyCollection(schema.columnInfo);
+      ICollection IDictionary.Values => new BlueRowValueCollection(this);
+
+      int ICollection.Count => schema.columnInfo.Length;
      bool IDictionary.IsFixedSize => true;
      bool IDictionary.IsReadOnly => true;
      object ICollection.SyncRoot => this;
      bool ICollection.IsSynchronized => false;
+      bool IDictionary.Contains(object key) {
+         if (key is string s) {
+            return schema.columnNames.Contains(s);
+         }
+         return false;
+      }
+      void ICollection.CopyTo(Array array, int index) {
+         var length = schema.columnInfo.Length;
+         for (int i = 0; i < length; i++) {
+            array.SetValue(this[i], i + index);
+         }
+      }
+      void IDictionary.Add(object key, object? value) => throw new NotImplementedException();
+      void IDictionary.Clear() => throw new NotImplementedException();
+      IDictionaryEnumerator IDictionary.GetEnumerator() => new BlueRowEnumerator(this);
+      IEnumerator IEnumerable.GetEnumerator() => ((IDictionary)this).GetEnumerator();
+      void IDictionary.Remove(object key) => throw new NotSupportedException();
   }

   // public interface IBlueRow<T> : IBlueRow where T : IBlueRow {
@@ -16,11 +16,23 @@ namespace BCG
      IBlueRow this[int id]{get;}
      IBlueTableSchema schema {get;}
      bool containsID (int id);
+
+      bool ICollection.IsSynchronized => false;
+      object ICollection.SyncRoot => this;
+
+      void ICollection.CopyTo(Array array, int index) {
+         foreach (IBlueRow r in this) {
+            array.SetValue(r, index++);
+         }
+      }
   }

   public interface IBlueRowCollection<T> : IBlueRowCollection, IReadOnlyCollection<T> where T : IBlueRow {
      new T this[int id] { get; }

      IBlueRow IBlueRowCollection.this[int id] => this[id];
+
+      IEnumerator IEnumerable.GetEnumerator() =>
+         GetEnumerator();
   }
 }
@@ -3,6 +3,8 @@ using System.Collections;
 using System.Data;
 using BCG.Utilities.General;

+using Bts.DataStreamCore;
+
 namespace BCG
 {
 	[Serializable]
@@ -183,6 +185,7 @@ namespace BCG
            return retVal;
         }
      }
+      IBlueTableSchema IDataPacket.Schema => schema;

      /*          
       status  D total  A total  D this   A This   D' total A' Total 
@@ -324,6 +327,12 @@ namespace BCG
      public IBlueRowCollection<T> newRows { get; }
      public IBlueRowCollection<T> addedRows { get; }
      public IBlueRowCollection<T> deletedRows { get; }
+
+      public static DataPacket<T> Empty(IBlueTableSchema schema) =>
+         new(
+            new BlueTableImmutable<T>(schema),
+            new BlueTableImmutable<T>(schema),
+            new BlueTableImmutable<T>(schema));
   }

   public static class DataPacketExtensions {
@@ -129,6 +129,12 @@ public class DataPacketJsonConverter : JsonConverter<DataPacket> {
         if (t == typeof(bool)) {
            return NullCheckWrapper(v => writer.WriteBooleanValue((bool)v));
         }
+         if (t == typeof(DateTime)) {
+            return NullCheckWrapper(v => writer.WriteStringValue(((DateTime)v).ToUniversalTime().ToString("yyyy-MM-ddTHH:mm:ss.ffffff")));
+         }
+         if (t == typeof(TimeSpan)) {
+            return NullCheckWrapper(v => writer.WriteStringValue(((TimeSpan)v).ToString(@"hh\:mm\:ss\.ffffff")));
+         }
         return NullCheckWrapper(v => writer.WriteStringValue(v?.ToString()));

         Action<object?> NullCheckWrapper(Action<object> writeNonNull) =>
@@ -0,0 +1,29 @@
+#nullable enable
+namespace Bts.DataStream;
+
+using BCG;
+
+using bts.SilverPine;
+
+using System.Collections.Generic;
+using System.Linq;
+
+public class CompositeDataStreamFactory : IDataStreamFactory {
+   readonly List<IDataStreamFactory> streamFactories;
+
+   public CompositeDataStreamFactory(IEnumerable<IDataStreamFactory> streamFactories) {
+      this.streamFactories = streamFactories.ToList();
+      foreach (var rsf in this.streamFactories.OfType<IRecursiveDataStreamFactory>()) {
+         rsf.OnCompositeFactory(this);
+      }
+   }
+
+   public IEnumerable<IDataStreamMetadata> GetSupportedStreams() =>
+      streamFactories.SelectMany(f => f.GetSupportedStreams());
+
+   public ViewStream? Create(string streamId, IEnumerable<(string? Name, int? Index, object? Value)> args) =>
+      streamFactories.Where(f => f.CanCreate(streamId))
+         .Select(f => f.Create(streamId, args))
+         .ExceptNull()
+         .FirstOrDefault();
+}
@@ -0,0 +1,52 @@
+#nullable enable
+namespace bts.SilverPine;
+
+using BCG;
+
+using System;
+using System.Collections.Generic;
+using System.Linq;
+using System.Reflection;
+
+public interface IDataStreamMetadata {
+   public string Name { get; }
+   public IEnumerable<string> AlternateNames { get; }
+}
+
+public class DataStreamMetadata : IDataStreamMetadata {
+   public string? Name { get; set; }
+   public List<string>? AlternateNames { get; set; }
+
+   string IDataStreamMetadata.Name => Name ?? string.Empty;
+   IEnumerable<string> IDataStreamMetadata.AlternateNames =>
+      AlternateNames ?? new(0);
+
+   public static DataStreamMetadata FromType(Type type) {
+      var friendlyName = type.GetCustomAttribute<DataStreamUserRespresentation>()?.friendlyName;
+      var rv = new DataStreamMetadata() {
+         Name = friendlyName ?? type.Name,
+         AlternateNames = new(2) {
+            Log.notnull(type.FullName)
+         }
+      };
+      if (friendlyName is not null) {
+         rv.AlternateNames.Add(type.Name);
+      }
+      return rv;
+   }
+}
+
+public static class DataStreamMetadataExtensions {
+   public static bool IsMatch(this IDataStreamMetadata dsm, string name, bool checkAlternateNames = true, bool ignoreCase = true, bool ignoreUnderscores = true) {
+      var strComp = ignoreCase ? StringComparison.CurrentCultureIgnoreCase : StringComparison.CurrentCulture;
+      var nameWithoutUnderscores = new Lazy<string>(() => RemoveUnderscores(name));
+      return IsNameMatch(dsm.Name) || (checkAlternateNames && dsm.AlternateNames.Any(IsNameMatch));
+
+      static string RemoveUnderscores(string s) =>
+         s.Replace("_", "");
+
+      bool IsNameMatch(string otherName) =>
+         string.Equals(name, otherName, strComp)
+            || (ignoreUnderscores && string.Equals(nameWithoutUnderscores.Value, RemoveUnderscores(otherName), strComp));
+   }
+}
@@ -0,0 +1,27 @@
+#nullable enable
+namespace Bts.DataStream;
+
+using BCG;
+
+using bts.SilverPine;
+
+using System.Collections.Generic;
+using System.Linq;
+
+public interface IDataStreamFactory {
+   IEnumerable<IDataStreamMetadata> GetSupportedStreams();
+
+   ViewStream? Create(string streamId, IEnumerable<(string? Name, int? Index, object? Value)> args);
+
+   bool CanCreate(string streamId) =>
+      GetSupportedStreams().Any(s => s.IsMatch(streamId));
+}
+
+/// <summary>
+/// This interface provides a way for IDataStreamFactory's that create ViewStreams
+/// that depend specifically on the composite IDataStreamFactory. This was designed
+/// specifically for the NLStreamDataFactory.
+/// </summary>
+public interface IRecursiveDataStreamFactory : IDataStreamFactory {
+   void OnCompositeFactory(IDataStreamFactory compositeFactory);
+}
@@ -9,31 +9,36 @@ namespace BCG {
   using System.Reflection;
   using System.IO;
   using System.Collections.Generic;
+   using System.Linq;
+   using bts.collections;
+   using System.Diagnostics.CodeAnalysis;
+   using bts.SilverPine;
+   using Spark.Core.Collections;
+   using Bts.DataStreamCore;
+   using Microsoft.Extensions.Options;

-   public class PrimitiveManager {
+   public class PrimitiveManager : IDataStreamFactory {
      private static PrimitiveManager mySelf;
      readonly List<string> myAssemblyNames = new();
      readonly Dictionary<string, Type> myTypeLookupTable = new();

-      private PrimitiveManager() {
-         myLoadAllTypes();
-      }
+      public PrimitiveManager(IOptions<PrimitiveManagerConfig> config) : this(config.Value) { }

-      private void myLoadAllTypes() {
+      public PrimitiveManager(PrimitiveManagerConfig config) {
         AppDomain currentDomain = AppDomain.CurrentDomain;
         currentDomain.AssemblyResolve += new ResolveEventHandler(myAssemblyResolver);
         loadAssembly(this.GetType().Assembly.FullName);
-         var assemblies = new string[] { "DataStream.dll", "SharedStreams.dll" };
-         var config_assemblies = Configuration.get<DataStreamConfig>().Assemblies;
-         bts.Log.Gen.info($"PrimitiveManager.myLoadAllTypes config_assemblies: {config_assemblies}");
-         foreach (string assemblyName in assemblies/*config_assemblies*/) {
+         foreach (string assemblyName in config.Assemblies.empty_if_null()) {
            bts.Log.Gen.info($"PrimitiveManager.myLoadAllTypes assemblyName: {assemblyName}");
            string resolvedAssemblyPath = assemblyName;
            if (!System.IO.Path.IsPathRooted(assemblyName)) {
               resolvedAssemblyPath = Path.GetFullPath(Path.Combine(AppDomain.CurrentDomain.BaseDirectory, assemblyName));
            }
-            loadAssembly(resolvedAssemblyPath);
+            loadAssembly(resolvedAssemblyPath, IncludeType);
         }
+
+         bool IncludeType(Type t) =>
+            !config.ExcludeTypes.empty_if_null().Any(tname => tname == t.Name);
      }

      private static Assembly myAssemblyResolver(object sender, ResolveEventArgs args) {
@@ -47,7 +52,12 @@ namespace BCG {
      /// Loads all the Stream types associated with the assembly
      /// </summary>
      /// <param name="assemblyName"></param>
-      public void loadAssembly(string assemblyName) {
+      public void loadAssembly(string assemblyName, Func<Type, bool> typeFilter = null) {
+         Func<Type, bool> is_silverpine_only = (t) => {
+            if (!assemblyName.Contains("SharedStreams.dll"))
+               return true;
+            return t.Namespace == "bts.SilverPine";
+         };
         bts.Log.Gen.info($"PrimitiveManager.loadAssembly assemblyName: {assemblyName}");
         if (myAssemblyNames.Contains(assemblyName))
            return;
@@ -56,8 +66,12 @@ namespace BCG {
            GetType().Assembly :
            Assembly.LoadFrom(assemblyName);

-         foreach (Type type in loadedAssembly.GetTypes())
-            if (!type.IsInterface
+         IEnumerable<Type> types = loadedAssembly.GetTypes();
+         if (typeFilter is not null) {
+            types = types.Where(typeFilter);
+         }
+         foreach (Type type in types)
+            if (!type.IsInterface && is_silverpine_only(type)
               && !type.IsAbstract
               && type.IsSubclassOf(typeof(ViewStream))
               && type.IsPublic) {
@@ -73,7 +87,7 @@ namespace BCG {
                     break;
                  }
               if (isVisible) {
-                  Log.assert_d(myTypeLookupTable[userRep] == null, type.FullName);
+                  Log.assert_d(!myTypeLookupTable.ContainsKey(userRep));
                  myTypeLookupTable[userRep] = type;
               }
            }
@@ -89,22 +103,30 @@ namespace BCG {
         return retVal;
      }

+      public IEnumerable<Type> GetSupportedTypes() =>
+         myTypeLookupTable.Values.Cast<Type>();

+      public IEnumerable<IDataStreamMetadata> GetSupportedStreams() =>
+         GenericDictionaryWrapper.Create<string, Type>(myTypeLookupTable).Select(kv => new DataStreamMetadata() {
+            Name = kv.Key,
+            AlternateNames = new() { kv.Value.Name, kv.Value.FullName }
+         });

-      public static PrimitiveManager instance {
-         get { return mySelf = mySelf == null ? new PrimitiveManager() : mySelf; }
+      public Type ResolveType(string streamId) =>
+         TryResolveType(streamId, out var t) ? t
+            : throw new ArgumentException($"Failed to resolve ViewStream type '{streamId}'", nameof(streamId));
+
+      public bool TryResolveType(string streamId, [NotNullWhen(true)] out Type result) {
+         result = GetSupportedStreams()
+            .Where(s => s.IsMatch(streamId))
+            .Select(s => s.Name)
+            .Prepend(streamId)
+            .Select(myTypeLookupTable.GetValueOrDefault)
+            .ExceptNull()
+            .FirstOrDefault();
+         return result is not null;
      }

-      /// <summary>
-      /// Returns the Type corresponding to the primitive name
-      /// </summary>
-      /// <param name="friendlyName"></param>
-      /// <returns></returns>
-      public Type find(string friendlyName) =>
-         myTypeLookupTable.TryGetValue(friendlyName, out var t)
-            ? t
-            : throw new ArgumentException($"'{friendlyName}' is not the friendly name of any of the loaded types", nameof(friendlyName));
-
      /// <summary>
      /// Constructs and returns a ViewStream of type corresponding to the friendly name
      /// </summary>
@@ -118,8 +140,110 @@ namespace BCG {
            constructorArgs = new object[args.Count];
            args.CopyTo(constructorArgs, 0);
         }
-         var type = find(friendlyName);
-         return Activator.CreateInstance(type, constructorArgs) as ViewStream;
+         if (myTypeLookupTable[friendlyName] == null)
+            throw new ApplicationException(friendlyName + " is not the friendly name of any of the loaded types");
+         try {
+            return
+               constructorArgs == null ?
+               Activator.CreateInstance(myTypeLookupTable[friendlyName] as Type) as ViewStream :
+               Activator.CreateInstance(myTypeLookupTable[friendlyName] as Type, constructorArgs) as ViewStream;
+         } catch (Exception ex) {
+            Log.Gen.warn($"PrimitiveManager failed to construct ViewStream type '{friendlyName}': {ex}");
+         }
+         return null;
+      }
+
+      public ViewStream Create(string streamId, IEnumerable<(string Name, int? Index, object Value)> args) =>
+         TryResolveType(streamId, out var type) ? Create(type, args) : null;
+
+      /// <summary>
+      /// Constructs and returns a ViewStream of type corresponding to the friendly name
+      /// </summary>
+      /// <param name="friendlyName">friendly primitive name </param>
+      /// <param name="args">args can be null or an empty ICollection if the paramterless constructor
+      /// is the desired one</param>
+      /// <returns></returns>
+      public ViewStream Create(Type type, IEnumerable<(string Name, int? Index, object Value)> args) {
+         var ctrs = type.GetConstructors(BindingFlags.Instance | BindingFlags.Public);
+         var argList = args.ToList();
+         const int MAX_ALLOWED_ARGS = 64;
+         if (argList.Count > MAX_ALLOWED_ARGS) {
+            throw new ArgumentException(
+               $"ViewStream arg count ({argList.Count}) exceeds max allowed ({MAX_ALLOWED_ARGS})", nameof(args));
+         }
+         if (argList.Any(a => !(a.Name?.Length > 0) && !a.Index.HasValue)) {
+            throw new ArgumentException(
+               "One or more ViewStream args lack both Name and Index (unable to assign a constructor param)", nameof(args));
+         }
+         var indexedArgs = argList.Where(a => a.Index.HasValue).ToList();
+         var namedArgs = argList.Where(a => a.Name?.Length > 0).ToList();
+         if (!namedArgs.Select(a => a.Name).IsDistinct()) {
+            throw new ArgumentException("ViewStream arg names are not unique");
+         }
+         if (!indexedArgs.Select(a => a.Index.Value).IsDistinct()) {
+            throw new ArgumentException("ViewStream arg indices are not unique");
+         }
+         // Constructor params with default values may be omitted from the input 'args'. Therefore,
+         // 'minArgCount' >= 'argList.Count'.
+         var minArgCount = Math.Max(indexedArgs.Select(a => a.Index.Value).MaxOrDefault(), argList.Count);
+         foreach (var ctr in ctrs) {
+            if (TryAssignConstructorArgs(ctr, minArgCount, argList, indexedArgs, namedArgs, out var ctrArgs)) {
+               return ctr.Invoke(ctrArgs) as ViewStream;
+            }
+         }
+         return null;
+
+         static bool TryAssignConstructorArgs(
+            ConstructorInfo ctr,
+            int minArgCount,
+            List<(string Name, int? Index, object Value)> argList,
+            List<(string Name, int? Index, object Value)> indexedArgs,
+            List<(string Name, int? Index, object Value)> namedArgs,
+            [NotNullWhen(true)] out object[] ctrArgs
+         ) {
+            ctrArgs = null;
+            var ctrParams = ctr.GetParameters();
+            if (ctrParams.Length < minArgCount) {
+               return false;
+            }
+            var ctrArgList = new List<object>();
+            int remInputArgs = argList.Count;
+            for (int i = 0; i < ctrParams.Length; i++) {
+               var ctrParam = ctrParams[i];
+               var paramType = ctrParams[i].ParameterType;
+               if (indexedArgs.TryGetFirst(a => CanAssignInputArg(ctrParam, a), out var a)) {
+                  remInputArgs--;
+                  ctrArgList.Add(a.Value);
+               } else if (ctrParam.HasDefaultValue) {
+                  ctrArgList.Add(ctrParam.DefaultValue);
+               } else {
+                  return false;
+               }
+            }
+            if (remInputArgs > 0) {
+               return false;
+            }
+            Log.assert(ctrArgList.Count == ctrParams.Length);
+            ctrArgs = ctrArgList.ToArray();
+            return true;
+         }
+
+         static bool CanAssignInputArg(ParameterInfo pi, (string Name, int? Index, object Value) arg) {
+            if (!string.IsNullOrEmpty(arg.Name) && arg.Name != pi.Name) {
+               return false;
+            }
+            if (arg.Index.HasValue && arg.Index.Value != pi.Position) {
+               return false;
+            }
+            if (arg.Value is null) {
+               if (!pi.ParameterType.IsValueType && !pi.ParameterType.IsNullable()) {
+                  return false;
+               }
+            } else if (!pi.ParameterType.IsAssignableFrom(arg.Value.GetType())) {
+               return false;
+            }
+            return true;
+         }
      }

      /// <summary>
@@ -0,0 +1,9 @@
+#nullable enable
+namespace Bts.DataStreamCore;
+
+using System.Collections.Generic;
+
+public class PrimitiveManagerConfig {
+   public List<string>? Assemblies { get; set; }
+   public List<string>? ExcludeTypes { get; set; }
+}
@@ -9,6 +9,7 @@ namespace BCG
 {
   public interface IDataPacket
   {
+      IBlueTableSchema Schema { get; }
      //this is an ICollection of IDictionary
      void get_new_added_deleted(out ICollection new_rows, out ICollection added_rows, out ICollection deleted_rows);
   }
@@ -18,6 +19,8 @@ namespace BCG
      IBlueRowCollection<T> addedRows { get; }
      IBlueRowCollection<T> deletedRows { get; }

+      IBlueTableSchema IDataPacket.Schema => newRows.schema;
+
      void IDataPacket.get_new_added_deleted(out ICollection new_rows, out ICollection added_rows, out ICollection deleted_rows) {
         new_rows = newRows;
         added_rows = addedRows;
@@ -3,6 +3,9 @@ namespace Bts.DataStreamCore.Services;
 using BCG;
 using BCG.Utilities.General;

+using Bts.DataStream;
+
+using Microsoft.Extensions.Configuration;
 using Microsoft.Extensions.DependencyInjection;

 public static class DataStreamCoreServices {
@@ -14,4 +17,11 @@ public static class DataStreamCoreServices {
         ViewStreamDatabase.SetGlobalInstance(new ViewStreamDatabase(sp.GetRequiredService<ISubscriberCore>()));
         return ViewStreamDatabase.instance;
      });
+
+   public static IServiceCollection AddPrimitiveDataStreamFactory(this IServiceCollection services, IConfiguration configurationSection) =>
+      services.Configure<PrimitiveManagerConfig>(configurationSection)
+         .AddSingleton<IDataStreamFactory, PrimitiveManager>();
+
+   public static IServiceCollection AddCompositeDataStreamFactory(this IServiceCollection services) =>
+      services.AddSingleton<CompositeDataStreamFactory>();
 }
@@ -2,15 +2,40 @@ namespace Bts.DataStreamExpressions;

 public class Argument {
   public Argument() { }
-   public Argument(string str) => String = str;
-   public Argument(bool? boolean) => Boolean = boolean;
-   public Argument(int? integer) => Integer = integer;
-   public Argument(double? real) => Real = real;
-   public Argument(DateTime? dateTime) => DateTime = dateTime;
-   public Argument(DBNull? @null) => Null = @null;
-   public Argument(StatementExpression? statementExp) => StatementExp = statementExp;
-   public Argument(QuotedView? quotedView) => QuotedView = quotedView;
+   public Argument(string str, string? name = null) {
+      String = str;
+      Name = name;
+   }
+   public Argument(bool? boolean, string? name = null) {
+      Boolean = boolean;
+      Name = name;
+   }
+   public Argument(int? integer, string? name = null) {
+      Integer = integer;
+      Name = name;
+   }
+   public Argument(double? real, string? name = null) {
+      Real = real;
+      Name = name;
+   }
+   public Argument(DateTime? dateTime, string? name = null) {
+      DateTime = dateTime;
+      Name = name;
+   }
+   public Argument(DBNull? @null, string? name = null) {
+      Null = @null;
+      Name = name;
+   }
+   public Argument(StatementExpression? statementExp, string? name = null) {
+      StatementExp = statementExp;
+      Name = name;
+   }
+   public Argument(QuotedView? quotedView, string? name = null) {
+      QuotedView = quotedView;
+      Name = name;
+   }

+   public string? Name { get; }
   public string? String { get; }
   public bool? Boolean { get; }
   public int? Integer { get; }
@@ -2,6 +2,8 @@ namespace Bts.DataStreamExpressions;

 using BCG;

+using Bts.DataStream;
+
 class Assignment : StatementExpression {
   readonly StatementExpression expression;
   readonly string id;
@@ -19,8 +21,8 @@ class Assignment : StatementExpression {
      return result;
   }

-   public override ViewStream? myGetView(IDictionary<string, StatementExpression> env) {
-      var result = expression.getView(env);
+   public override ViewStream? myGetView(IDictionary<string, StatementExpression> env, IDataStreamFactory streamFactory) {
+      var result = expression.getView(env, streamFactory);
      env[id] = expression;
      return result;
   }
@@ -2,6 +2,8 @@ namespace Bts.DataStreamExpressions;

 using BCG;

+using Bts.DataStream;
+
 using System.Text;

 public class DataStreamExpression {
@@ -31,10 +33,10 @@ public class DataStreamExpression {
      return sb.ToString();
   }

-   public ViewStream? getView(IDictionary<string, StatementExpression> env) {
+   public ViewStream? getView(IDictionary<string, StatementExpression> env, IDataStreamFactory streamFactory) {
      for (int i = 0; i < Statements.Count - 1; i++) {
-         Statements[i].getView(env);
+         Statements[i].getView(env, streamFactory);
      }
-      return Statements[^1].getView(env);
+      return Statements[^1].getView(env, streamFactory);
   }
 }
@@ -22,7 +22,7 @@
 }

 %token BOOLEAN_CONSTANT, INTEGER_CONSTANT, REAL_CONSTANT, DATE_TIME_CONSTANT, CS_STRING_CONSTANT, NULL_CONSTANT
-%token PARENTHESIS_OPEN, PARENTHESIS_CLOSE, COMMA, IDENTIFIER, EQUALS, QUOTE_OPEN, QUOTE_CLOSE
+%token PARENTHESIS_OPEN, PARENTHESIS_CLOSE, COMMA, COLON, IDENTIFIER, EQUALS, QUOTE_OPEN, QUOTE_CLOSE
 %token LINE_COMMENT

 %nonassoc EQUALS
@@ -72,13 +72,21 @@ Arguments :

 Argument :
   CS_STRING_CONSTANT { $$ = new($1.CsString); }
+   | IDENTIFIER COLON CS_STRING_CONSTANT { $$ = new($3.CsString, $1.Identifier); }
   | BOOLEAN_CONSTANT { $$ = new($1.Boolean); }
+   | IDENTIFIER COLON BOOLEAN_CONSTANT { $$ = new($3.Boolean, $1.Identifier); }
   | INTEGER_CONSTANT { $$ = new($1.Integer); }
+   | IDENTIFIER COLON INTEGER_CONSTANT { $$ = new($3.Integer, $1.Identifier); }
   | REAL_CONSTANT { $$ = new($1.Real); }
+   | IDENTIFIER COLON REAL_CONSTANT { $$ = new($3.Real, $1.Identifier); }
   | DATE_TIME_CONSTANT { $$ = new($1.DateTime); }
+   | IDENTIFIER COLON DATE_TIME_CONSTANT { $$ = new($3.DateTime, $1.Identifier); }
   | NULL_CONSTANT { $$ = new($1.Null); }
+   | IDENTIFIER COLON NULL_CONSTANT { $$ = new($3.Null, $1.Identifier); }
   | Statement { $$ = new($1); }
+   | IDENTIFIER COLON Statement { $$ = new($3, $1.Identifier); }
   | QuotedView { $$ = new($1); }
+   | IDENTIFIER COLON QuotedView { $$ = new($3, $1.Identifier); }
   ;

 QuotedView :
@@ -31,6 +31,7 @@ Identifier               {Letter}({Letter}*[0-9]*_*)*
 OpenParenthesis          \(
 CloseParenthesis         \)
 Comma                    ,
+Colon                    :
 QuoteOpen                \[
 QuoteClose               \]

@@ -90,6 +91,10 @@ LineComment \/\/[^\n\r]*(\n|\r|\r\n)
   return (int)DataStreamExpressionToken.COMMA;
 }

+{Colon} {
+   return (int)DataStreamExpressionToken.COLON;
+}
+
 {QuoteOpen} {
   return (int)DataStreamExpressionToken.QUOTE_CLOSE;
 }
@@ -2,7 +2,8 @@ namespace Bts.DataStreamExpressions;

 using BCG;

-using System.Collections;
+using Bts.DataStream;
+
 using System.Collections.Immutable;

 public class PrimitiveExpression : StatementExpression {
@@ -20,7 +21,7 @@ public class PrimitiveExpression : StatementExpression {
   }

   public override string print(int startCol, IList<string> activeBindings) {
-      string prelude = "new " + PrimitiveManager.instance.find(myName).Name + " (";
+      string prelude = "new " + myName + " (";

      return prelude + myPrintArgs(startCol + prelude.Length, activeBindings) + ")";
   }
@@ -31,21 +32,19 @@ public class PrimitiveExpression : StatementExpression {
      return prelude + myPrintViewArgs(startCol + prelude.Length) + ")";
   }

-   public override ViewStream myGetView(IDictionary<string, StatementExpression> env) {
-      ArrayList resolvedArgs = new ArrayList();
+   public override ViewStream? myGetView(IDictionary<string, StatementExpression> env, IDataStreamFactory streamFactory) {
+      var resolvedArgs = myArgs.Select((arg, i) => ((string?)arg.Name, string.IsNullOrEmpty(arg.Name) ? (int?)i : null, ResolveArg(arg)));
+      return streamFactory.Create(myName, resolvedArgs);

-      foreach (var arg in myArgs) {
+      object? ResolveArg(Argument arg) {
         if (arg.StatementExp is not null) {
-            resolvedArgs.Add(arg.StatementExp.myGetView(env));
-            continue;
+            return arg.StatementExp.myGetView(env, streamFactory);
         }
         if (arg.QuotedView is not null) {
-            resolvedArgs.Add(arg.QuotedView.Value);
-            continue;
+            return arg.QuotedView.Value;
         }
-         resolvedArgs.Add(arg.Obj);
+         return arg.Obj;
      }
-      return PrimitiveManager.instance.build(myName, resolvedArgs);
   }

   string myPrintSingleArg(int atColumn, object? o, IList<string> activeBindings) {
@@ -2,6 +2,8 @@ namespace Bts.DataStreamExpressions;

 using BCG;

+using Bts.DataStream;
+
 public abstract class StatementExpression {
   ViewStream? view;

@@ -13,10 +15,10 @@ public abstract class StatementExpression {
   public virtual string? printView(int startCol) =>
      null;

-   public ViewStream? getView(IDictionary<string, StatementExpression> env) =>
-      view ??= myGetView(env);
+   public ViewStream? getView(IDictionary<string, StatementExpression> env, IDataStreamFactory streamFactory) =>
+      view ??= myGetView(env, streamFactory);

-   public virtual ViewStream? myGetView(IDictionary<string, StatementExpression> env) =>
+   public virtual ViewStream? myGetView(IDictionary<string, StatementExpression> env, IDataStreamFactory streamFactory) =>
      null;

   public static string makeCSString(string s) {
@@ -2,6 +2,8 @@ namespace Bts.DataStreamExpressions;

 using BCG;

+using Bts.DataStream;
+
 public class ViewNameExpression : StatementExpression {
   readonly string id;

@@ -14,6 +16,6 @@ public class ViewNameExpression : StatementExpression {
   public override string printView(int startCol) =>
      id;

-   public override ViewStream? myGetView(IDictionary<string, StatementExpression> env) =>
-      env.TryGetValue(id, out var exp) ? exp.getView(env) : new TableStream(id);
+   public override ViewStream? myGetView(IDictionary<string, StatementExpression> env, IDataStreamFactory streamFactory) =>
+      env.TryGetValue(id, out var exp) ? exp.getView(env, streamFactory) : new TableStream(id);
 }
@@ -1,8 +1,8 @@
 FROM bluetradingsystems/linux_dotnet7

-ARG CS_SOURCE_DIR="build/Release/bin/net7.0/linux-x64/publish"
-ARG CPP_SOURCE_DIR="install/data-stream-suite-linux"
-ARG CONF_FILE="src/DataStreamSuite/DataStreamPublisher/spark.conf"
+ARG DS_PUBLISHER_CS_SOURCE_DIR="build/Release/bin/net7.0/linux-x64/publish"
+ARG DS_PUBLISHER_CPP_SOURCE_DIR="install/data-stream-suite-linux"
+ARG DS_PUBLISHER_CONF_FILE="src/DataStreamSuite/DataStreamPublisher/spark.conf"

 ENV DOCKERIZE_VERSION=v0.7.0
 USER root
@@ -12,7 +12,7 @@ RUN apt-get update \
    && apt-get autoremove -yqq --purge wget && rm -rf /var/lib/apt/lists/*

 USER bts
-COPY --chown=bts:bts "${CPP_SOURCE_DIR}/lib" "bin"
-COPY --chown=bts:bts "${CS_SOURCE_DIR}" "bin"
-COPY --chown=bts:bts "${CONF_FILE}" ".local/share/bts/DataStreamPublisher/conf/spark.conf"
+COPY --chown=bts:bts "${DS_PUBLISHER_CPP_SOURCE_DIR}/lib" "bin"
+COPY --chown=bts:bts "${DS_PUBLISHER_CS_SOURCE_DIR}" "bin"
+COPY --chown=bts:bts "${DS_PUBLISHER_CONF_FILE}" ".local/share/bts/DataStreamPublisher/conf/spark.conf"
 WORKDIR "/home/bts/bin"
@@ -4,6 +4,7 @@ using BCG;

 using bts;

+using Bts.DataStream;
 using Bts.DataStreamCore;
 using Bts.DataStreamExpressions;
 using Bts.PubSub;
@@ -28,13 +29,28 @@ public class DataStreamPublisherService : IHostedService, IDisposable {
   readonly DataStreamPublisherConfig config;
   readonly IPubTableBroker pubTableBroker;
   readonly IWorkScheduler workScheduler;
+   readonly IDataStreamFactory streamFactory;
   List<IDisposable>? resources = new();

-   public DataStreamPublisherService(IPubTableBroker pubTableBroker, IWorkScheduler workScheduler, IOptions<DataStreamPublisherConfig> config)
-      : this(pubTableBroker, workScheduler, config.Value) { }
+   public DataStreamPublisherService(
+      IPubTableBroker pubTableBroker,
+      IEnumerable<IDataStreamFactory> streamFactories,
+      IWorkScheduler workScheduler,
+      IOptions<DataStreamPublisherConfig> config
+   )
+      : this(pubTableBroker, streamFactories, workScheduler, config.Value) { }

-   public DataStreamPublisherService(IPubTableBroker pubTableBroker, IWorkScheduler workScheduler, DataStreamPublisherConfig config) {
+   public DataStreamPublisherService(
+      IPubTableBroker pubTableBroker,
+      IEnumerable<IDataStreamFactory> streamFactories,
+      IWorkScheduler workScheduler,
+      DataStreamPublisherConfig config
+   ) {
      this.pubTableBroker = pubTableBroker;
+      var streamFactoryList = streamFactories.ToList();
+      streamFactory = streamFactoryList.Count == 1
+         ? streamFactoryList[0]
+         : new CompositeDataStreamFactory(streamFactoryList);
      this.workScheduler = workScheduler;
      this.config = config;
   }
@@ -48,7 +64,7 @@ public class DataStreamPublisherService : IHostedService, IDisposable {
               Log.Gen.warn("Skipping PublishedStream because it doesn't have a name (must define 'Name' in config json).");
               continue;
            }
-            var vs = CreateViewStream(publishedStream);
+            var vs = CreateViewStream(streamFactory, publishedStream);
            if (vs is null) {
               continue;
            }
@@ -72,7 +88,7 @@ public class DataStreamPublisherService : IHostedService, IDisposable {
      resources?.Clear();
   }

-   static ViewStream? CreateViewStream(PublishedStream ps) {
+   static ViewStream? CreateViewStream(IDataStreamFactory streamFactory, PublishedStream ps) {
      Log.Gen.info(
         $"Instantiating ViewPublisher for: {JsonSerializer.Serialize(ps)}");
      if (string.IsNullOrEmpty(ps.Name)) {
@@ -80,7 +96,7 @@ public class DataStreamPublisherService : IHostedService, IDisposable {
            $"Failed to instantiate ViewPublisher. Null/empty 'name': {JsonSerializer.Serialize(ps)}");
         return null;
      }
-      var vs = buildStream(ps.Expression);
+      var vs = buildStream(streamFactory, ps.Expression);
      if (vs is null) {
         Log.Gen.warn(
            $"Failed to instantiate ViewPublisher. Invalid 'expression': {JsonSerializer.Serialize(ps)}");
@@ -89,14 +105,14 @@ public class DataStreamPublisherService : IHostedService, IDisposable {
      return vs;
   }

-   static ViewStream? buildStream(string? streamExpression) {
+   static ViewStream? buildStream(IDataStreamFactory streamFactory, string? streamExpression) {
      if (streamExpression is null) {
         return null;
      }
      try {
         Log.Gen.info($"Building ViewStream: {streamExpression}");
         var exp = DataStreamExpressionsInterpreter.Evaluate(streamExpression);
-         return exp.getView(new Dictionary<string, StatementExpression>(0));
+         return exp.getView(new Dictionary<string, StatementExpression>(0), streamFactory);
      } catch (Exception ex) {
         Log.assert(false, $"Could not build stream. expression: {streamExpression}; ex: {ex}");
         return null;
@@ -19,6 +19,7 @@ public static class DataStreamPublisherServices {
         .AddPubSubClient(configurationSection.GetSection("PubSub"))
         .AddPubTableBroker()
         .AddRemoteQueryManager()
+         .AddPrimitiveDataStreamFactory(configurationSection.GetSection("PrimitiveStream"))
         .Configure<DataStreamPublisherConfig>(configurationSection.GetSection("DataStreamPublisher"))
         .AddHostedService<DataStreamPublisherService>();
 }
@@ -12,6 +12,16 @@
  "ViewStreamDatabase": {
    "Interval": "00:00:00.1"
  },
+  "PrimitiveStream": {
+    "Assemblies": [
+      "DataStream.dll",
+      "SharedStreams.dll"
+    ],
+    "ExcludedTypes": [
+      "SettlementPriceStream",
+      "OHLCVStream"
+    ]
+  },
  "PubSub": {
    "HeartbeatInterval": 15000,
    "MissingHeartbeatsBeforeGc": 50,
@@ -44,6 +44,8 @@ Project("{FAE04EC0-301F-11D3-BF4B-00C04F79EFBC}") = "Bts.Extensions.Hosting", "B
 EndProject
 Project("{FAE04EC0-301F-11D3-BF4B-00C04F79EFBC}") = "Bts.Net", "Bts.Net\Bts.Net.csproj", "{D92E1456-91C1-43FA-AA87-67BFE663944F}"
 EndProject
+Project("{FAE04EC0-301F-11D3-BF4B-00C04F79EFBC}") = "PandasExpressions", "PandasExpressions\PandasExpressions.csproj", "{32F45D7C-6DE5-4660-AD45-EDEE3B41E26F}"
+EndProject
 Global
 	GlobalSection(SolutionConfigurationPlatforms) = preSolution
 		Debug|Any CPU = Debug|Any CPU
@@ -137,5 +139,9 @@ Global
 		{D92E1456-91C1-43FA-AA87-67BFE663944F}.Debug|Any CPU.Build.0 = Debug|Any CPU
 		{D92E1456-91C1-43FA-AA87-67BFE663944F}.Release|Any CPU.ActiveCfg = Release|Any CPU
 		{D92E1456-91C1-43FA-AA87-67BFE663944F}.Release|Any CPU.Build.0 = Release|Any CPU
+		{32F45D7C-6DE5-4660-AD45-EDEE3B41E26F}.Debug|Any CPU.ActiveCfg = Debug|Any CPU
+		{32F45D7C-6DE5-4660-AD45-EDEE3B41E26F}.Debug|Any CPU.Build.0 = Debug|Any CPU
+		{32F45D7C-6DE5-4660-AD45-EDEE3B41E26F}.Release|Any CPU.ActiveCfg = Release|Any CPU
+		{32F45D7C-6DE5-4660-AD45-EDEE3B41E26F}.Release|Any CPU.Build.0 = Release|Any CPU
 	EndGlobalSection
 EndGlobal
@@ -0,0 +1,114 @@
+namespace Bts.DataStreamTests;
+
+using BCG;
+
+using Bts.DataStreamCore;
+
+using Spark.Core;
+
+[TestFixture]
+public class BlueColumnTableTests {
+   static IBlueTableSchema defaultSchema = new BlueTableSchemaBuilder("addTestTable") {
+         ("id", typeof(int)),
+         ("name", typeof(string)),
+         ("value", typeof(double)),
+      }
+      .ToSchema();
+
+   static BlueColumnTable getTable(params Variant[][] rows) {
+      var bct = new BlueColumnTable(defaultSchema);
+      for (int i = 0; i < rows.Length; i++) {
+         bct.Add(rows[i]);
+      }
+      return bct;
+   }
+
+   [Test]
+   public void TestAdd() {
+      var rows = new Variant[][] {
+         new Variant[] { 1, "Joe", 32 },
+         new Variant[] { 2, "Angie", 26.5 },
+      };
+      var bct = getTable();
+      Is.EqualTo(bct.Count == 0);
+      bct.Add(rows[0]);
+      bct.Add(rows[1]);
+      Is.EqualTo(bct.Count == 2);
+      assertRowEquals(bct[rows[0][0].GetInt32()], rows[0]);
+      assertRowEquals(bct[rows[1][0].GetInt32()], rows[1]);
+      assertRowSequenceEqual(bct, rows);
+      rows[0] = new Variant[] { 1, "Mika", 4.2 };
+      bct.AddOrUpdate(rows[0]);
+      Is.EqualTo(bct.Count == 2);
+      assertRowEquals(bct[rows[0][0].GetInt32()], rows[0]);
+   }
+
+   [Test]
+   public void TestRemove() {
+      var rows = new Variant[][] {
+         new Variant[] { 5, "Angie", 26.5 },
+         new Variant[] { 1, "Joe", 32 },
+         new Variant[] { 8, "Myrtle", 103 },
+      };
+      var bct = getTable(rows);
+      bct.Remove(rows[1][0].GetInt32());
+      assertRowSequenceEqual(bct, new[] { rows[0], rows[2] });
+   }
+
+   [Test]
+   public void TestRemoveReAdd() {
+      var rows = new Variant[][] {
+         new Variant[] { 5, "Angie", 26.5 },
+         new Variant[] { 1, "Joe", 32 },
+         new Variant[] { 8, "Myrtle", 103 },
+      };
+      var bct = getTable(rows);
+      bct.Remove(rows[1][0].GetInt32());
+      bct.Add(rows[1]);
+      assertRowSequenceEqual(bct, rows);
+   }
+
+   [Test]
+   public void TestClear() {
+      var rows = new Variant[][] {
+         new Variant[] { 5, "Angie", 26.5 },
+         new Variant[] { 1, "Joe", 32 },
+         new Variant[] { 8, "Myrtle", 103 },
+      };
+      var bct = getTable(rows);
+      bct.Clear();
+      assertRowSequenceEqual(bct, Array.Empty<Variant[]>());
+   }
+
+   static void assertRowEquals(IBlueRow row, Variant[] expectedValues, IBlueTableSchema? schema = null) {
+      schema ??= defaultSchema;
+      for (int i = 0; i < schema.columnInfo.Length; i++) {
+         Is.EqualTo(Equals(row[schema.columnInfo[i].colNo], expectedValues[i]));
+         if (!schema.columnInfo.TryGetFirst(c => c!.colNo == i, out var col)) {
+            throw new InvalidOperationException($"Schema has no column at index {i}");
+         }
+         Is.EqualTo(Equals(row[col.colName], expectedValues[i]));
+      }
+   }
+
+   static void assertRowSequenceEqual(BlueColumnTable bct, IEnumerable<Variant[]> rowValues) {
+      if (!bct.schema.columnInfo.TryGetFirst(c => c!.colName == "id", out var idCol)) {
+         throw new ArgumentException("Schema has no id column", nameof(bct));
+      }
+      Assert.That(bct.Cast<IBlueRow>().SequenceEqual(
+         rowValues.OrderBy(r => r[idCol.colNo].GetInt32()).Select(r => new MyRow(r, bct.schema)),
+         BlueRowEqualityComparer.Instance));
+   }
+
+   class MyRow : IBlueRow {
+      readonly Variant[] values;
+
+      public MyRow(Variant[] values, IBlueTableSchema schema) {
+         this.values = values;
+         this.schema = schema;
+      }
+
+      public IBlueTableSchema schema { get; }
+      public object? this[int index] => values[index].GetValue();
+   }
+}
@@ -0,0 +1,18 @@
+<Project>
+  
+  <Import Project="..\Directory.Build.targets" />
+
+  <Target
+    Name="GenerateRowTypes"
+    BeforeTargets="BeforeCompile"
+    Condition="'@(RowType)'!=''"
+    Inputs="%(RowType.SchemaJson);%(RowType.Identity)"
+    Outputs="$(IntermediateOutputPath)/RowTypes/%(RowType.TypeName).cs">
+    <MakeDir Directories="$(IntermediateOutputPath)/RowTypes" />
+    <Exec Command="dotnet t4 %(RowType.Identity) -p SchemaJson=%(RowType.SchemaJson) -p TypeName=%(RowType.TypeName) -p TableName=%(RowType.TableName) -o $(IntermediateOutputPath)/RowTypes/%(RowType.TypeName).cs" />
+    <ItemGroup>
+      <Compile Include="$(IntermediateOutputPath)/RowTypes/%(RowType.TypeName).cs" />
+    </ItemGroup>
+  </Target>
+
+</Project>
@@ -0,0 +1,6 @@
+namespace PandasExpressions;
+
+public class Class1
+{
+
+}
@@ -0,0 +1,72 @@
+%namespace Bts.DataStreamExpressions
+%partial
+%visibility internal
+%tokentype DataStreamExpressionToken
+%parsertype DataStreamExpressionParser
+
+%union {
+   public int Integer;
+   public double Real;
+   public bool Boolean;
+   public None None;
+   public string Identifier;
+   public DataFrameExpression DataFrame;
+   public StatementExpression Statement;
+   public PrimitiveExpression Primitive;
+   public ImmutableList<Argument> Arguments;
+   public Argument Argument;
+}
+
+%token BOOLEAN_CONSTANT, INTEGER_CONSTANT, REAL_CONSTANT, NONE_CONSTANT
+%token PARENTHESIS_OPEN, PARENTHESIS_CLOSE, BRACKET_OPEN, BRACKET_CLOSE, DOT, COMMA, COLON, IDENTIFIER, EQUALS
+%token AND, OR, PLUS, MINUS, TIMES, DIVIDE
+
+%nonassoc EQUALS
+
+%type <DataFrame> DataFrame
+%type <Statement> Statement
+%type <Argument> Argument
+%type <Arguments> Arguments
+
+%%
+
+ArithmeticColumnExpression :
+
+DataFrameExpression :
+   IDENTIFIER O
+
+Arguments :
+   {
+      $$ = System.Collections.Immutable.ImmutableList<Argument>.Empty;
+   }
+   | Argument {
+      $$ = System.Collections.Immutable.ImmutableList<Argument>.Empty.Add($1);
+   }
+   | Arguments COMMA Argument {
+      $$ = $1.Add($3);
+   }
+   ;
+
+ColumnExpression :
+   IDENTIFIER DOT IDENTIFIER {
+      $$ = new ColumnNameExpression($1, $3);
+   }
+   | IDENTIFIER OPEN_BRACKET IDENTIFIER CLOSE_BRACKET {
+      $$ = new ColumnNameExpression($1, $3);
+   }
+   | ColumnExpression PLUS ColumnExpression {
+      $$ = $1.Plus($3);
+   }
+   | ColumnExpression MINUS ColumnExpression {
+      $$ = $1.Minus($3);
+   }
+   | ColumnExpression TIMES ColumnExpression {
+      $$ = $1.Multiply($3);
+   }
+   | ColumnExpression DIVIDE ColumnExpression {
+         
+   }
+   | ColumnExpression AND ColumnExpression {
+         
+   }
+   | ColumnExpression OR ColumnExpression
@@ -0,0 +1,93 @@
+%namespace Bts.PandasExpressions
+%scannertype PandasExpressionScanner
+%visibility internal
+%tokentype PandasExpressionToken
+
+%option stack, minimize, parser, verbose, persistbuffer, noembedbuffers
+
+Identifier               [a-zA-Z_][a-zA-Z_0-9]*
+Equals                   =
+IntegerConstant          -?[0-9]+
+RealConstant             -?[0-9]+\.[0-9]+
+BooleanConstant          True|False
+And                      &&
+Or                       ||
+None                     None
+OpenParenthesis          \(
+CloseParenthesis         \)
+OpenBracket              \[
+CloseBracket             \]
+Dot                      .
+Comma                    ,
+Colon                    :
+Plus                     +
+Minus                    -
+Times                    *
+Divide                   /
+
+
+%%
+
+{Equals} {
+   return (int)PandasExpressionToken.EQUALS;
+}
+
+{BooleanConstant} {
+   yylval.Boolean = yytext.ToLower() switch {
+      "True" => true,
+      "False" => false,
+      _ => throw new Exception(
+         string.Format("Expected yytext to be 'True' or 'False' (actually '{0}')", yytext))
+   };
+   return (int)PandasExpressionToken.BOOLEAN_CONSTANT;
+}
+
+{IntegerConstant} {
+   yylval.Integer = int.Parse(yytext);
+   return (int)PandasExpressionToken.INTEGER_CONSTANT;
+}
+
+{RealConstant} {
+   yylval.Real = double.Parse(yytext);
+   return (int)PandasExpressionToken.REAL_CONSTANT;
+}
+
+{None} {
+   yylval.None = None.Value;
+   return (int)PandasExpressionToken.NONE_CONSTANT;
+}
+
+{OpenParenthesis} {
+   return (int)PandasExpressionToken.PARENTHESIS_OPEN;
+}
+
+{CloseParenthesis} {
+   return (int)PandasExpressionToken.PARENTHESIS_CLOSE;
+}
+
+{Dot} {
+   return (int)PandasExpressionToken.DOT;
+}
+
+{Comma} {
+   return (int)PandasExpressionToken.COMMA;
+}
+
+{Colon} {
+   return (int)PandasExpressionToken.COLON;
+}
+
+{OpenBracket} {
+   return (int)PandasExpressionToken.BRACKET_OPEN;
+}
+
+{CloseBracket} {
+   return (int)PandasExpressionToken.BRACKET_CLOSE;
+}
+
+{Identifier} {
+   yylval.Identifier = yytext;
+   return (int)PandasExpressionToken.IDENTIFIER;
+}
+
+%%
@@ -0,0 +1,9 @@
+<Project Sdk="Microsoft.NET.Sdk">
+
+  <PropertyGroup>
+    <TargetFramework>net7.0</TargetFramework>
+    <ImplicitUsings>enable</ImplicitUsings>
+    <Nullable>enable</Nullable>
+  </PropertyGroup>
+
+</Project>
@@ -182,9 +182,9 @@ namespace bts
            }
            n_transfers++;
            total_bytes_read += bytes_read;
-            Log.Gen.info($"AsyncTCPConnection_AsyncWrapper._read_fixed_size_msg n_bytes: {n_bytes} bytes_read: {bytes_read} total_bytes_read: {total_bytes_read} buff_len: {buffer.Length}");
+            Log.DataFlow.diag($"AsyncTCPConnection_AsyncWrapper._read_fixed_size_msg n_bytes: {n_bytes} bytes_read: {bytes_read} total_bytes_read: {total_bytes_read} buff_len: {buffer.Length}");
         }
-         Log.Gen.info($"AsyncTCPConnection_AsyncWrapper._read_fixed_size_msg read {n_bytes} bytes in {n_transfers} transfers");
+         Log.DataFlow.diag($"AsyncTCPConnection_AsyncWrapper._read_fixed_size_msg read {n_bytes} bytes in {n_transfers} transfers");
         return buffer;
      }

@@ -223,13 +223,13 @@ namespace bts
         BinaryReader br = new BinaryReader(new MemoryStream(sz_arr));
         br.BaseStream.Position = 0;
         int msg_sz = br.ReadInt32();
-         Log.Gen.info($"AsyncTCPConnection_AsyncWrapper._read_one_message read msg_sz: {msg_sz}");
+         Log.DataFlow.diag($"AsyncTCPConnection_AsyncWrapper._read_one_message read msg_sz: {msg_sz}");
         if (msg_sz < 0)
         {
            throw new IOException("Read 0 bytes, the connection was closed before all data could be read.");
         }
         byte[] msg = _read_fixed_size_msg(msg_sz);
-         Log.Gen.info($"AsyncTCPConnection_AsyncWrapper._read_one_message read msg sz: {msg.Length}");
+         Log.DataFlow.diag($"AsyncTCPConnection_AsyncWrapper._read_one_message read msg sz: {msg.Length}");
         // TODO, read first 4 bytes and decide hb or data
         OneRecvMessage rv = new OneRecvMessage(msg);
         return rv;
@@ -336,4 +336,4 @@ namespace bts
         return _connection_id.ToString();
      }
   }
-}
+}
@@ -19,14 +19,16 @@ namespace bts {
   /// </summary>
   public class SingleTagData {
      private readonly IPubSubServerTagHandler? _tag_handler;
+      readonly HashSet<string> noisyTags = new() {
+         "pss_hb"
+      };

      public SingleTagData(string tag, string tag_type, IPubSubServerTagHandler tag_handler) {
-         bts.Log.Gen.info($"SingleTagData constructed for tag: {tag} tag_type: {tag_type}");
+         Log.Gen.info($"SingleTagData.cons tag: {tag} tag_type: {tag_type} tag_handler: {tag_handler}");
         this.tag = tag;
         this.tag_type = tag_type;
         _tag_handler = tag_handler;
         Log.notnull(_tag_handler);
-         Log.Gen.info($"SingleTagData.cons tag: {tag} tag_type: {tag_type} _tag_handler: {_tag_handler}");
      }

      public string tag { get; }
@@ -36,7 +38,9 @@ namespace bts {
      // PubsubState calls this
      public void handle_publish(PublishMsg pub_msg) {
         string handler_str = _tag_handler != null ? _tag_handler.GetType().ToString() : "NULL";
-         bts.Log.Gen.info($"SingleTagData.handle_publish tag: {tag} pub_msg: {pub_msg} handler: {handler_str} ");
+         if (pub_msg.tag is not null && !noisyTags.Contains(pub_msg.tag)) {
+            bts.Log.Gen.info($"SingleTagData.handle_publish tag: {tag} pub_msg: {pub_msg} handler: {handler_str} ");
+         }
         try {
            // tag_handler may be null. If it throws (a null reference ex), we'll catch and handle
            // it like any other ex.
@@ -9,8 +9,90 @@ namespace bts.SilverPine {
   using ZstdSharp.Unsafe;

   public static class ContractUtils {
+      public class FutureTradableContract {
+         public string symbol;
+         public DateTime expiration;
+         public Exchange exchange;
+         public string contract_text;
+      }
+      public class FutureSpreadContract {
+         public List<Tuple<double, bts.SilverPine.ContractUtils.FutureTradableContract>> legs;
+         public string symbol {
+            get {
+               return legs == null || legs.Count == 0 ? String.Empty : legs.ElementAt(0).Item2.symbol;
+            }
+         }
+         public string contract_text;
+      }
+      public class FutureContinousContract {
+         public string symbol;
+         public Exchange exchange;
+      }
+      // this is for all future conytracts belonging to a symbol
+      public class FutureWildcardContract {
+         public string symbol;
+         public Exchange exchange;
+      }
+      public class FutureContract {
+         public FutureWildcardContract wildcard_contract = null;
+         public FutureTradableContract future_tradable_contract = null;
+         public FutureSpreadContract spread_tradable_contract = null;
+         public FutureContinousContract continous_constract = null;
+
+         public FutureContract(string txt) {
+            wildcard_contract = parse_wildcard_contract(txt);
+            if (wildcard_contract != null)
+               return;
+            FutureTradableContract fc = parse_future_contract(txt);
+            if (fc != null) {
+               future_tradable_contract = fc;
+               return;
+            }
+            FutureSpreadContract sc = parse_future_spread_contract(txt);
+            if (sc != null) {
+               spread_tradable_contract = sc;
+               return;
+            }
+            // this is not totally true, since ParseFutureSpreadContract does not
+            // parse ICS spreads
+            continous_constract = new FutureContinousContract() { symbol = txt };
+         }
+         public bool is_continous_contract => continous_constract != null;
+         public string symbol() {
+            if (wildcard_contract != null)
+               return wildcard_contract.symbol;
+            if (continous_constract != null) {
+               return continous_constract.symbol;
+            } else if (future_tradable_contract != null) {
+               return future_tradable_contract.symbol;
+            } else if (spread_tradable_contract != null) {
+               return spread_tradable_contract.symbol;
+            }
+            Log.assert(false);
+            return String.Empty;
+         }
+         public string contract_text {
+            get {
+               if (future_tradable_contract != null)
+                  return future_tradable_contract.contract_text;
+               if (spread_tradable_contract != null)
+                  return spread_tradable_contract.contract_text;
+               return null;
+            }
+         }
+      }
      #region Parsing
-      public static FutureContract ParseFutureContract(string contract_txt, Exchange exchange = Exchange.CME) {
+      public static FutureWildcardContract parse_wildcard_contract(string contract_txt, Exchange exchange = Exchange.CME) {
+         if (contract_txt.EndsWith("*")) {
+            contract_txt = contract_txt.Replace("*", String.Empty);
+            return new FutureWildcardContract() {
+               symbol = contract_txt,
+               exchange = exchange
+            };
+         }
+         return null;
+      }
+      public static FutureTradableContract parse_future_contract(string contract_txt, Exchange exchange = Exchange.CME) {
         // Validate the future string length
         if (!contract_txt.All(c => Char.IsLetterOrDigit(c)))
            return null;
@@ -56,23 +138,24 @@ namespace bts.SilverPine {

         // Create the FutureContract instance and set its properties
         DateTime expiration = new DateTime(year, month, 1);
-         var futureContract = new FutureContract {
+         var futureContract = new FutureTradableContract {
            symbol = symbol,
            expiration = expiration,
-            exchange = exchange
+            exchange = exchange,
+            contract_text = contract_txt
         };

         return futureContract;
      }
      // Method to parse a futures spread, e.g. "CLX4-CLZ4"
-      public static FutureSpreadContract ParseFutureSpreadContract(string spreadString, Exchange exchange = Exchange.CME) {
+      public static FutureSpreadContract parse_future_spread_contract(string contract_txt, Exchange exchange = Exchange.CME) {
         // Split the string by '-' to handle each leg
-         var spreadParts = spreadString.Split('-');
+         var spreadParts = contract_txt.Split('-');
         if (spreadParts.Length < 2) {
            return null; // Not a valid spread format
         }

-         var spreadContract = new FutureSpreadContract { legs = new List<Tuple<double, FutureContract>>() };
+         var spreadContract = new FutureSpreadContract { legs = new List<Tuple<double, FutureTradableContract>>() };

         foreach (var part in spreadParts) {
            // Split each leg by '*', assuming the format is weight*future or just future
@@ -92,33 +175,32 @@ namespace bts.SilverPine {
            }

            // Parse the future contract
-            var futureContract = ParseFutureContract(futurePart, exchange);
+            var futureContract = parse_future_contract(futurePart, exchange);
            if (futureContract == null) {
               return null; // Failed to parse the future part
            }

            // Add the leg to the spread contract
-            spreadContract.legs.Add(new Tuple<double, FutureContract>(weight, futureContract));
+            spreadContract.legs.Add(new Tuple<double, FutureTradableContract>(weight, futureContract));
         }
-
         return spreadContract;
      }

-      public static ContractType DetermineFutureOrSpread(string contract) {
-         if (contract.Contains("-")) {
-            return ContractType.FutureSpread;
-         } else {
-            return ContractType.Future;
-         }
-      }
+      // public static ContractType DetermineFutureOrSpread(string contract) {
+      //    if (contract.Contains("-")) {
+      //       return ContractType.FutureSpread;
+      //    } else {
+      //       return ContractType.Future;
+      //    }
+      // }
      #endregion

      public static ContractType get_type(string contract_txt) {
-         FutureContract fc = ParseFutureContract(contract_txt);
+         FutureTradableContract fc = parse_future_contract(contract_txt);
         if (fc != null) {
            return ContractType.Future;
         }
-         FutureSpreadContract sc = ParseFutureSpreadContract(contract_txt);
+         FutureSpreadContract sc = parse_future_spread_contract(contract_txt);
         if (sc != null) {
            return ContractType.FutureSpread;
         }
@@ -133,11 +215,11 @@ namespace bts.SilverPine {
      }

      public static Tuple<ContractCategory, string> get_symbol_category(string symbol) {
-         ContractUtils.FutureContract fc = ParseFutureContract(symbol);
+         ContractUtils.FutureTradableContract fc = parse_future_contract(symbol);
         if (fc != null) {
            return Tuple.Create(ContractCategory.FutureString, fc.symbol);
         }
-         var sc = ParseFutureSpreadContract(symbol);
+         var sc = parse_future_spread_contract(symbol);
         if (sc != null) {
            return Tuple.Create(ContractCategory.FutureString, sc.symbol);
         }
@@ -165,18 +247,6 @@ namespace bts.SilverPine {
      public enum Exchange {
         CME
      }
-      public class FutureContract {
-         public string symbol;
-         public DateTime expiration;
-         public Exchange exchange;
-      }
-      public class FutureSpreadContract {
-         public List<Tuple<double, bts.SilverPine.ContractUtils.FutureContract>> legs;
-         public string symbol {
-            get {
-               return legs == null || legs.Count == 0 ? String.Empty : legs.ElementAt(0).Item2.symbol;
-            }
-         }
-      }
+
   }
 }
@@ -1,36 +1,107 @@
-namespace BCG {
+namespace bts.SilverPine {
   using BCG.Utilities.General;

-   using bts.api;
-   using bts.api.domain;
-   using bts.api.synthetic;
   using bts.md;
-   using bts.spark;
-   using bts.utils;
+   using BCG;

-   using mbts;
-   using mbts.co2;
-   using mbts.config;
-   using mbts.spark;
-   using mbts.ui;
-
-   using Spark.Core.Data;
   using Spark.Core.Disposables;
   using Spark.Core.Reactive;

   using System;
-   using System.Collections.Concurrent;
   using System.Collections.Generic;
-   using System.Collections.ObjectModel;
   using System.Linq;
   using System.Reactive.Linq;
   using bts.api.core;
   using Spark.Api.MarketData;

   using SecurityKey = System.UInt32;
-   using System.Data;
-   using System.Reflection.Metadata;
-   using bts;
+   using System.Data.Entity.Infrastructure;
+
+   public class FakeBBOStream : ViewStream, IWorker {
+      private string _symbol;
+      public BlueTable _table = new BlueTable(BBOStream._bbo_schema);
+      private DateTime _last_fired_time = DateTime.MinValue;
+      private TimeSpan _period = TimeSpan.FromMilliseconds(333);
+      private double _start_price;
+      private double _inc;
+      private int _total_count = 5;
+      private Random _rnd = new Random(DateTime.Now.Millisecond);
+      private string[] _months = new string[]{
+         "F","G", "H", "J","K", "M", "N","Q","U","V","X","Z"
+      };
+      private ViewStreamDatabase _vsd;
+      public FakeBBOStream(string contract_type, string exchange, string symbol, double start_price, double inc)
+      :
+      this(contract_type, exchange, symbol, start_price, inc, ViewStreamDatabase.instance) { }
+
+      public FakeBBOStream(string contract_type, string exchange, string symbol, double start_price, double inc, ViewStreamDatabase vsd) {
+         _symbol = symbol;
+         _start_price = start_price;
+         _inc = inc;
+         _vsd = vsd;
+         Log.assert(contract_type.ToLower() == "future");
+      }
+
+      protected override void myOnFirstSubscriber() {
+         _vsd.addWorker(this);
+      }
+
+      protected override void myOnLastSubscriber() {
+         _vsd.removeWorker(this);
+      }
+
+      private bool _random_truefase() {
+         int r = _rnd.Next(0, 100);
+         return r % 2 == 0;
+      }
+
+      private IBlueRow _perturb_row(IBlueRow er) {
+         MutableBlueRow rv = new MutableBlueRow(er);
+         double bid_price = rv.get<double>("bid_price");
+         bid_price += _inc;
+         double ask_price = bid_price + _inc;
+         rv.setValue("bid_price", bid_price);
+         rv.setValue("ask_price", ask_price);
+         return rv;
+      }
+
+      public void doWork() {
+         if (DateTime.Now - _last_fired_time < _period)
+            return;
+         _last_fired_time = DateTime.Now;
+         if (_table.Count == 0) {
+            // first fire_symbol
+            string year_code = DateTime.Now.Year.ToString().Last().ToString();
+            for (int i = 0; i < _total_count; i++) {
+               string month_code = _months[i % _months.Length];
+               string sec = $"{_symbol}{month_code}{year_code}";
+               double bid_price = _start_price + _rnd.Next(-10, 10) * _inc;
+               double ask_price = bid_price + _inc;
+               BlueRow nr = new BlueRow(_table.schema, _table.Count, _table.Count, sec,
+                                       bid_price, _rnd.Next(1, 10),
+                                       ask_price, _rnd.Next(1, 10));
+               _table.addRow(nr);
+               var dp = BlueDataUtility.build_dp(new BlueTable(_table), myLastValue as DataPacket);
+               myFireEvent(dp);
+            }
+         } else {
+            BlueTable deleted_rows = new BlueTable(_table.schema);
+            BlueTable added_rows = new BlueTable(_table.schema);
+            // go through all rows, select some at random, add them to deleted, get
+            // the replaced one and add it to added, updates table state and fire
+            for (int i = 0; i < _table.Count; i++) {
+               if (_random_truefase()) {
+                  IBlueRow er = _table[i];
+                  IBlueRow nr = _perturb_row(er);
+                  _table.deleteRow(er);
+                  _table.addRow(nr);
+               }
+            }
+            var dp = BlueDataUtility.build_dp_inc(new BlueTable(_table), myLastValue as DataPacket, true);
+            myFireEvent(dp);
+         }
+      }
+   }

   [DataStreamUserRespresentation("bbo")]
   public class BBOStream : ViewStream, IWorker {
@@ -40,7 +111,7 @@ namespace BCG {
      private int _id = 0;
      IReadOnlyDictionary<SecurityKey, BookTuple> _cached_state = null;
      MarketDataClient _md_client;
-      private readonly IBlueTableSchema _bbo_schema = BlueDataUtility.form_schema("bbo",
+      internal static IBlueTableSchema _bbo_schema = BlueDataUtility.form_schema("bbo",
                                                Tuple.Create("id", TypeCode.Int32),
                                                Tuple.Create("sec_key", TypeCode.UInt32),
                                                Tuple.Create("sec", TypeCode.String),
@@ -56,9 +127,11 @@ namespace BCG {
      }

      public BBOStream(ViewStreamDatabase vsd) {
-         bts.Log.Gen.info("BBOStream.cons");
-         _vsd = vsd;
-         _md_client = new MarketDataClient(new MDSource(TimeSpan.FromSeconds(0.5)));
+         // bts.Log.Gen.info("BBOStream.cons");
+         // _vsd = vsd;
+         // _md_client = new MarketDataClient(new MDSource(TimeSpan.FromSeconds(0.5)));
+         FakeBBOStream rs = new FakeBBOStream("Future", "CME", "ES", 5000, 0.25, vsd);
+         mySetRealSource(rs);
      }

      protected override void myOnFirstSubscriber() {
@@ -0,0 +1,241 @@
+namespace bts.SilverPine {
+   using BCG;
+
+   using System.Runtime.CompilerServices;
+
+   using ZstdSharp.Unsafe;
+
+   public class DateTimeRange {
+      public readonly DateTime range_beg;
+      public readonly DateTime range_end;
+      public DateTimeRange(DateTime range_beg_, DateTime range_end_) {
+         range_beg = range_beg_;
+         range_end = range_end_;
+      }
+      public static DateTimeRange parse(string range_beg_, string range_end_) {
+         DateTime range_beg = String.IsNullOrEmpty(range_beg_) ? DateTime.MinValue : DateTime.Parse(range_beg_);
+         DateTime range_end = String.IsNullOrEmpty(range_end_) ? DateTime.MaxValue : DateTime.Parse(range_end_);
+         return new DateTimeRange(range_beg, range_end);
+      }
+   }
+
+   public class FutureBarsLoader {
+      public static string get_db_table_name(BarSize bar_size, string symbol, bool is_continous) {
+         switch (bar_size) {
+            case BarSize.ONE_DAY: {
+                  if (is_continous) {
+                     return $"bars_future_{symbol}_continous_1day";
+                  } else
+                     return $"bars_future_{symbol}_all_1day";
+               }
+            case BarSize.ONE_MIN: {
+                  if (is_continous) {
+                     return $"bars_future_{symbol}_continous_1min";
+                  } else
+                     return $"bars_future_{symbol}_all_1min";
+               }
+         }
+         Log.assert(false);
+         return null;
+      }
+   }
+   public enum BarSize {
+      ONE_DAY = 0,
+      ONE_MIN
+   }
+
+   [DataStreamUserRespresentation("FuturesHistorical1DayBars")]
+   public class FuturesHistorical1DayBarsStream : ViewStream {
+
+      // this gets the raw bars for a symbol (perhaps make it into a raw bars stream?)
+      public FuturesHistorical1DayBarsStream(string symbol) :
+      this(symbol, DateTime.MinValue, DateTime.MaxValue) { }
+
+      // creates the non raw stream (ie for tradable contract, or continous contract)
+      public FuturesHistorical1DayBarsStream(string future_contract, DateTime range_beg, DateTime range_end)
+     : this(new ContractUtils.FutureContract(future_contract), new DateTimeRange(range_beg, range_end)) { }
+
+      // this creates a tradable contracts bars  streams or continous contracts stream. if get_all_rows is true, 
+      // it gets the raw table with all contracts for that symbol
+      public FuturesHistorical1DayBarsStream(ContractUtils.FutureContract future_contract,
+                                             DateTimeRange dt_range) {
+         DateTime beg = dt_range.range_beg;//.ToUniversalTime(),
+         DateTime end = dt_range.range_end;//.ToUniversalTime();
+         beg = beg.Date;
+         end = end.Date.Add(new TimeSpan(23, 59, 59));
+         // this returns rows from beg and end inclusive
+         Func<string, SQLiteStream> get_sqlite_str = (table_name) => {
+            string range_beg_iso8601 = beg.ToString("yyyy-MM-ddTHH:mm:ss"),
+               range_end_iso8601 = end.ToString("yyyy-MM-ddTHH:mm:ss");
+            string sql_str = $"select * from {table_name} where ts_event >= '{range_beg_iso8601}' AND ts_event <= '{range_end_iso8601}'";
+            return new SQLiteStream(SQLiteHelper.db_name, sql_str);
+         };
+         string symbol = future_contract.symbol();
+         if (future_contract.wildcard_contract != null) {
+            string table_name = FutureBarsLoader.get_db_table_name(BarSize.ONE_DAY, future_contract.symbol(), false);
+            ViewStream rs = get_sqlite_str(table_name);
+            //rs = new DateStandardizationStream(rs, "ts_event", DateConversion.ChopTime);
+            mySetRealSource(rs);
+            return;
+         }
+         if (future_contract.wildcard_contract != null) {
+            string table_name = FutureBarsLoader.get_db_table_name(BarSize.ONE_DAY, future_contract.symbol(), false);
+            ViewStream rs = get_sqlite_str(table_name);
+            //rs = new DateStandardizationStream(rs, "ts_event", DateConversion.ChopTime);
+            mySetRealSource(rs);
+            return;
+         }
+         if (future_contract.is_continous_contract) {
+            string table_name = FutureBarsLoader.get_db_table_name(BarSize.ONE_DAY, symbol, true);
+            ViewStream rs = get_sqlite_str(table_name);
+            //rs = new DateStandardizationStream(rs, "ts_event", DateConversion.ChopTime);
+            mySetRealSource(rs);
+            return;
+         } else {
+            string table_name = FutureBarsLoader.get_db_table_name(BarSize.ONE_DAY, future_contract.symbol(), false);
+            ViewStream rs = get_sqlite_str(table_name);
+            //rs = new DateStandardizationStream(rs, "ts_event", DateConversion.ChopTime);
+            //TODO: filter it 
+            rs = new FilterStream(rs, $"symbol = '{future_contract.contract_text}'");
+            mySetRealSource(rs);
+         }
+      }
+   }
+
+   [DataStreamUserRespresentation("FuturesHistorical1MinBars")]
+   public class FuturesHistorical1MinuteBarsStream : ViewStream {
+      // same as the above but for one min
+      // this gets the raw bars for a symbol
+      public FuturesHistorical1MinuteBarsStream(string symbol) :
+      this(symbol, DateTime.MinValue, DateTime.MaxValue) { }
+
+      // creates the non raw stream (ie for tradable contract, or continous contract)
+      public FuturesHistorical1MinuteBarsStream(string future_contract, DateTime range_beg, DateTime range_end)
+     : this(new ContractUtils.FutureContract(future_contract), new DateTimeRange(range_beg, range_end)) { }
+
+      // this creates a tradable contracts bars  streams or continous contracts stream. if get_all_rows is true, 
+      // it gets the raw table with all contracts for that symbol
+      public FuturesHistorical1MinuteBarsStream(ContractUtils.FutureContract future_contract,
+                                             DateTimeRange dt_range) {
+         Func<string, SQLiteStream> get_sqlite_str = (table_name) => {
+            string range_beg_iso8601 = dt_range.range_beg.ToString("yyyy-MM-ddTHH:mm:ss"),
+               range_end_iso8601 = dt_range.range_end.ToString("yyyy-MM-ddTHH:mm:ss");
+            //string sql_str = $"select * from {table_name} where DATE(ts_event) >= '{range_beg_iso8601}' AND DATE(ts_event) <= '{range_end_iso8601}'";
+            string sql_str = $"select * from {table_name} where ts_event >= '{range_beg_iso8601}' AND ts_event <= '{range_end_iso8601}'";
+            return new SQLiteStream(SQLiteHelper.db_name, sql_str);
+         };
+         string symbol = future_contract.symbol();
+         if (future_contract.wildcard_contract != null) {
+            string table_name = FutureBarsLoader.get_db_table_name(BarSize.ONE_MIN, future_contract.symbol(), false);
+            ViewStream rs = get_sqlite_str(table_name);
+            mySetRealSource(rs);
+            return;
+         }
+         if (future_contract.is_continous_contract) {
+            string table_name = FutureBarsLoader.get_db_table_name(BarSize.ONE_MIN, symbol, true);
+            ViewStream rs = get_sqlite_str(table_name);
+            mySetRealSource(rs);
+            return;
+         } else {
+            string table_name = FutureBarsLoader.get_db_table_name(BarSize.ONE_MIN, future_contract.symbol(), false);
+            ViewStream rs = get_sqlite_str(table_name);
+            // TODO: filter
+            rs = new FilterStream(rs, $"symbol = '{future_contract.contract_text}'");
+            mySetRealSource(rs);
+         }
+      }
+   }
+
+   [DataStreamUserRespresentation("FutureHistoricalBars")]
+   public class FutureHistoricalBarsStream : ViewStream {
+      public FutureHistoricalBarsStream(string fut_contract, string range_beg, string range_end) {
+         DateTime dt_range_beg = DateTime.Parse(range_beg);
+         DateTime dt_range_end = DateTime.Parse(range_end);
+         TimeSpan diff = dt_range_end - dt_range_beg;
+         BarSize bar_size = diff < TimeSpan.FromDays(1.0) ? BarSize.ONE_MIN : BarSize.ONE_DAY;
+         ViewStream rs = new FutureHistoricalBarsStream(fut_contract, bar_size.ToString(), dt_range_beg, dt_range_end);
+         mySetRealSource(rs);
+      }
+      public FutureHistoricalBarsStream(string fut_contract, string bar_size,
+                                        DateTime dt_range_beg, DateTime dt_range_end)
+                                        :
+      this(new ContractUtils.FutureContract(fut_contract), Enum.Parse<BarSize>(bar_size, true),
+         new DateTimeRange(dt_range_beg, dt_range_end)) { }
+
+      public FutureHistoricalBarsStream(ContractUtils.FutureContract fut_contract, BarSize bar_size,
+                                        DateTimeRange dt_range) {
+         switch (bar_size) {
+            case BarSize.ONE_DAY: {
+                  ViewStream rs = new FuturesHistorical1DayBarsStream(fut_contract, dt_range);
+                  mySetRealSource(rs);
+                  break;
+               }
+            case BarSize.ONE_MIN: {
+                  ViewStream rs = new FuturesHistorical1MinuteBarsStream(fut_contract, dt_range);
+                  mySetRealSource(rs);
+                  break;
+               }
+         }
+      }
+   }
+
+   [DataStreamUserRespresentation("FuturePriceDiff")]
+   public class FutureHistoricalBarsPriceDiffStream : ViewStream {
+
+      public FutureHistoricalBarsPriceDiffStream(string future_contract, string dt1, string dt2)
+      :
+      this(new ContractUtils.FutureContract(future_contract), DateTime.Parse(dt1), DateTime.Parse(dt2), "close", ViewStreamDatabase.instance) {
+      }
+
+
+      public FutureHistoricalBarsPriceDiffStream(string future_contract, DateTime dt1, DateTime dt2)
+      :
+      this(new ContractUtils.FutureContract(future_contract), dt1, dt2, "close", ViewStreamDatabase.instance) {
+      }
+
+      public FutureHistoricalBarsPriceDiffStream(string future_contract, DateTime dt1, DateTime dt2, string col)
+      :
+      this(new ContractUtils.FutureContract(future_contract), dt1, dt2, col, ViewStreamDatabase.instance) {
+      }
+
+
+      public FutureHistoricalBarsPriceDiffStream(ContractUtils.FutureContract fc, DateTime dt1, DateTime dt2, string col, ViewStreamDatabase vsd) {
+         BarSize bsize = dt2 - dt1 >= TimeSpan.FromDays(1.0) ? BarSize.ONE_DAY : BarSize.ONE_MIN;
+         ViewStream rs = new FutureHistoricalBarsStream(fc, BarSize.ONE_DAY, new DateTimeRange(dt1, dt2));
+
+         rs = new ColumnDifferenceStream(rs, r => {
+            DateTime r_dt = r.get_datetime("ts_event");
+            var rv = dt1.Equals(r_dt);
+            return rv;
+         },
+         //r => dt2.Equals(r.get_datetime("ts_event")), col, col);
+         r => {
+            DateTime r_dt = r.get_datetime("ts_event");
+            var rv = dt2.Equals(r_dt);
+            return rv;
+         }, col, col);
+         rs = new NewProjectStream(rs, "diff !");
+         mySetRealSource(rs);
+      }
+   }
+
+   // takes the contract name and expands it into multuiple columns corresponding to the contracts.
+   // the contract columns are a union, of all properties with some of them empty
+   public class ContractExpanderStream : ViewStream {
+      private static IBlueTableSchema map_schema(IBlueTableSchema schema) {
+         return schema.add_column(TypeCode.String, "contract_type");
+      }
+      public ContractExpanderStream(ViewStream parent_str, string contract_col_name) {
+         ViewStream rs = new MapStream(parent_str, (schema) => map_schema(schema),
+         r => {
+            string contract = r.get<string>(contract_col_name);
+            var ct = ContractUtils.get_type(contract);
+            MutableBlueRow rv = new MutableBlueRow(map_schema(r.schema), r);
+            rv.setValue("contract_type", ct.ToString());
+            return rv;
+         });
+         mySetRealSource(rs);
+      }
+   }
+
+}
@@ -0,0 +1,46 @@
+using BCG;
+
+namespace bts.SilverPine {
+   /*
+      ----------------------------------------------------------------------
+                                  *Expression*
+      ----------------------------------------------------------------------
+          //r=FutureHistoricalBars("ES*", "02/01/2024 09:30 AM", "02/01/2024 09:30 PM")
+          r=FutureHistoricalBars("ES*","One_Day", 02/01/2024, 02/01/2025)
+          r=contractExpander(r,"symbol")
+          r=filter(r,"contract_type='FutureSpread'")
+          r=aggregate(r,"symbol","volume:Sum","ts_event")
+          r=aggregate(r,"volume_sum:Max")
+          r=newProject(r,"volume_sum_max:volume symbol !")
+
+      ----------------------------------------------------------------------
+                                    *Schema*
+      ----------------------------------------------------------------------
+          symbol                          System.String
+          id                              System.Int32
+          volume                          System.Double
+   */
+
+   [DataStreamUserRespresentation("VolumeAgg")]
+   public class VolumeAggregationStream : ViewStream {
+      public VolumeAggregationStream(string future_contract, string contract_type, DateTime range_beg, DateTime range_end)
+      : this(future_contract, contract_type, range_beg, range_end, ViewStreamDatabase.instance) { }
+
+      public VolumeAggregationStream(string future_contract, string contract_type, string range_beg, string range_end)
+      :
+      this(future_contract, contract_type, DateTime.Parse(range_beg), DateTime.Parse(range_end), ViewStreamDatabase.instance) { }
+
+      public VolumeAggregationStream(string future_contract, string contract_type, DateTime range_beg, DateTime range_end, ViewStreamDatabase db) {
+         future_contract = future_contract.EndsWith("*") ? future_contract : future_contract + "*";
+         ViewStream r;
+         r = new FutureHistoricalBarsStream(future_contract,
+                                             range_beg.ToString(), range_end.ToString());
+         r = new ContractExpanderStream(r, "symbol");
+         r = new FilterStream(r, $"contract_type='{contract_type}'");
+         r = new NonIncGroupByStream(r, "symbol", "volume:Sum", "ts_event");
+         r = new NonIncGroupByStream(r, "volume_sum:Max");
+         r = new NewProjectStream(r, "volume_sum_max:volume symbol !");
+         mySetRealSource(r);
+      }
+   }
+}
@@ -1,68 +1,117 @@
+// namespace bts.SilverPine {
+//    using bts;
+//    using BCG;
+//    using MongoDB.Driver.Linq;
+//    using System.ComponentModel.DataAnnotations;
+//    using ZstdSharp.Unsafe;
+
+//    [DataStreamUserRespresentation("VWAP")]
+//    public class VWAPStream : ViewStream {
+//       // returns the VWAP stream for each contract 
+//       private static ViewStream build_vwap_stream(string symbol, DateTime begin_dt, DateTime end_dt) {
+//          var ohlcv_params = OHLCVStream.get_params(begin_dt, end_dt);
+//          ViewStream ohlc_str = new OHLCVStream(symbol, ohlcv_params);
+//          // stream is grouped per contract and then vwap is calculated per contract
+//          GlobalMapStream rs = new GlobalMapStream(ohlc_str, "ts_event", "contract",
+//          (rows, idx) => {
+//             Log.Gen.info($"VWAPStream.GlobalMapStream delegate n_rows: {rows.Count} idx: {idx}");
+//             IBlueRow ir = rows.ElementAt((int)idx);
+//             var sel_rows = rows.Skip((int)idx).Where(r => r.get_datetime("ts_event") >= end_dt && r.get_datetime("ts_event") <= begin_dt);
+//             var volumeprices = sel_rows.Select(r => Tuple.Create(r.get<int>("volume"), r.get<double>("close"))).ToList();
+//             double vwap = IndicatorCalc.calculate_VWAP(volumeprices);
+//             IBlueTableSchema schema = ir.schema.add_columns(Tuple.Create(typeof(double), "vwap"));
+//             MutableBlueRow nr = new MutableBlueRow(schema, ir);
+//             //vwap = -313.987;
+//             nr.setValue("vwap", vwap);
+//             return nr;
+//          });
+//          return rs;
+//       }
+//       // the args want dt1 > dt2, if its nto we, swap them
+//       static void swap_if_necessary(ref DateTime dt1, ref DateTime dt2) {
+//          if (dt1 < dt2) {
+//             DateTime aux = dt1;
+//             dt1 = dt2;
+//             dt2 = aux;
+//          }
+//       }
+
+//       // this assumes the ohlc_str has diff contracts
+//       public VWAPStream(string symbol_or_contract, DateTime begin_dt, DateTime end_dt) {
+//          swap_if_necessary(ref begin_dt, ref end_dt);
+//          Tuple<ContractUtils.ContractCategory, string> ccat_sym_contr = ContractUtils.get_symbol_category(symbol_or_contract);
+//          string symbol = String.Empty;
+//          symbol = ccat_sym_contr.Item2;
+//          var ohlcv_params = OHLCVStream.get_params(begin_dt, end_dt);
+//          ViewStream vwap_str = build_vwap_stream(symbol, begin_dt, end_dt);
+//          // vwap_str now has vwap for all contracts. if the arg was a symbol, we need to filter it for the active contract,
+//          // whihc we do with FutureActivesContractStream. If the arg was a contract we'll simply filter it
+//          ViewStream rs = vwap_str;
+//          switch (ccat_sym_contr.Item1) {
+//             case ContractUtils.ContractCategory.FutureString:
+//             case ContractUtils.ContractCategory.SpreadString:
+//                rs = new FilterStream(vwap_str, r => r.get<string>("contract").CompareTo(ccat_sym_contr.Item2) == 0);
+//                break;
+//             case ContractUtils.ContractCategory.Symbol:
+//                rs = new FuturesActiveContractStream(vwap_str);
+//                break;
+//          }
+//          rs = new FilterStream(rs, r => r.get_datetime("ts_event").CompareTo(begin_dt) == 0);
+//          rs = new NewProjectStream(rs, "! vwap");
+//          mySetRealSource(rs);
+//       }
+
+//       public VWAPStream(string symbol_or_contract, string begin_dt, string end_dt)
+//       :
+//       this(symbol_or_contract, DateTime.Parse(begin_dt), DateTime.Parse(end_dt)) { }
+//    }
+// }
+
+using System;
+
+using BCG;
+
+using bts.api;
+
 namespace bts.SilverPine {
-   using bts;
-   using BCG;
-   using MongoDB.Driver.Linq;
-   using System.ComponentModel.DataAnnotations;
-   using ZstdSharp.Unsafe;
+   /*
+      ----------------------------------------------------------------------
+                                  *Expression*
+      ----------------------------------------------------------------------
+          //nl("Find Settlement price of the SP500 future in the first week of 2023.")
+
+          r=FutureHistoricalBars("ES", "One_Day", "1/1/2024", "2/1/2024")
+          r=math(r, "vprice = volume * close")
+          r=aggregate(r, "vprice:Sum, volume:Sum")
+          r=math(r, "vwap = vprice_sum / volume_sum")
+          r=newProject(r, "vwap !")
+
+      ----------------------------------------------------------------------
+                                    *Schema*
+      ----------------------------------------------------------------------
+          id                              System.Int32
+          vwap                            System.Double
+   */

   [DataStreamUserRespresentation("VWAP")]
   public class VWAPStream : ViewStream {
-      // returns the VWAP stream for each contract 
-      private static ViewStream build_vwap_stream(string symbol, DateTime begin_dt, DateTime end_dt) {
-         var ohlcv_params = OHLCVStream.get_params(begin_dt, end_dt);
-         ViewStream ohlc_str = new OHLCVStream(symbol, ohlcv_params);
-         // stream is grouped per contract and then vwap is calculated per contract
-         GlobalMapStream rs = new GlobalMapStream(ohlc_str, "ts_event", "contract",
-         (rows, idx) => {
-            Log.Gen.info($"VWAPStream.GlobalMapStream delegate n_rows: {rows.Count} idx: {idx}");
-            IBlueRow ir = rows.ElementAt((int)idx);
-            var sel_rows = rows.Skip((int)idx).Where(r => r.get_datetime("ts_event") >= end_dt && r.get_datetime("ts_event") <= begin_dt);
-            var volumeprices = sel_rows.Select(r => Tuple.Create(r.get<int>("volume"), r.get<double>("close"))).ToList();
-            double vwap = IndicatorCalc.calculate_VWAP(volumeprices);
-            IBlueTableSchema schema = ir.schema.add_columns(Tuple.Create(typeof(double), "vwap"));
-            MutableBlueRow nr = new MutableBlueRow(schema, ir);
-            //vwap = -313.987;
-            nr.setValue("vwap", vwap);
-            return nr;
-         });
-         return rs;
-      }
-      // the args want dt1 > dt2, if its nto we, swap them
-      static void swap_if_necessary(ref DateTime dt1, ref DateTime dt2) {
-         if (dt1 < dt2) {
-            DateTime aux = dt1;
-            dt1 = dt2;
-            dt2 = aux;
-         }
-      }
-
-      // this assumes the ohlc_str has diff contracts
-      public VWAPStream(string symbol_or_contract, DateTime begin_dt, DateTime end_dt) {
-         swap_if_necessary(ref begin_dt, ref end_dt);
-         Tuple<ContractUtils.ContractCategory, string> ccat_sym_contr = ContractUtils.get_symbol_category(symbol_or_contract);
-         string symbol = String.Empty;
-         symbol = ccat_sym_contr.Item2;
-         var ohlcv_params = OHLCVStream.get_params(begin_dt, end_dt);
-         ViewStream vwap_str = build_vwap_stream(symbol, begin_dt, end_dt);
-         // vwap_str now has vwap for all contracts. if the arg was a symbol, we need to filter it for the active contract,
-         // whihc we do with FutureActivesContractStream. If the arg was a contract we'll simply filter it
-         ViewStream rs = vwap_str;
-         switch (ccat_sym_contr.Item1) {
-            case ContractUtils.ContractCategory.FutureString:
-            case ContractUtils.ContractCategory.SpreadString:
-               rs = new FilterStream(vwap_str, r => r.get<string>("contract").CompareTo(ccat_sym_contr.Item2) == 0);
-               break;
-            case ContractUtils.ContractCategory.Symbol:
-               rs = new FuturesActiveContractStream(vwap_str);
-               break;
-         }
-         rs = new FilterStream(rs, r => r.get_datetime("ts_event").CompareTo(begin_dt) == 0);
-         rs = new NewProjectStream(rs, "! vwap");
-         mySetRealSource(rs);
-      }
-
-      public VWAPStream(string symbol_or_contract, string begin_dt, string end_dt)
+      public VWAPStream(string symbol_or_contract, DateTime beg_dt, DateTime end_dt)
      :
-      this(symbol_or_contract, DateTime.Parse(begin_dt), DateTime.Parse(end_dt)) { }
+      this(symbol_or_contract, beg_dt, end_dt, ViewStreamDatabase.instance) { }
+
+      public VWAPStream(string symbol_or_contract, DateTime beg_dt, DateTime end_dt, ViewStreamDatabase db) {
+         var fc = new ContractUtils.FutureContract(symbol_or_contract);
+         ViewStream r = new FutureHistoricalBarsStream(fc, BarSize.ONE_DAY, new DateTimeRange(beg_dt, end_dt));
+
+         r = new MathStream(r,
+                 "vprice = volume * close");
+         r = new NonIncGroupByStream(r,
+                                      "vprice:Sum, volume:Sum");
+         r = new MathStream(r,
+                             "vwap = vprice_sum / volume_sum");
+         r = new NewProjectStream(r,
+                                   "vwap !");
+         mySetRealSource(r);
+      }
   }
-}
+}
@@ -75,4 +75,36 @@ namespace bts.SilverPine {
         mySetRealSource(rs);
      }
   }
+}
+
+
+namespace bts.SilverPine {
+   using BCG;
+
+   using bts.api;
+
+   using Newtonsoft.Json.Serialization;
+
+   [DataStreamUserRespresentation("FuturesContinousContract")]
+   public class FuturesContinousContractStream : ViewStream {
+      // the argument is symbol, like "ES" etc and not the contract itself
+      public FuturesContinousContractStream(string symbol) : this(symbol, ViewStreamDatabase.instance) { }
+      public FuturesContinousContractStream(string symbol, ViewStreamDatabase db) {
+         ViewStream r = new FuturesHistorical1DayBarsStream(symbol);
+         // filtering for futures only
+         r = new FilterStream(r, r => {
+            string contract = r.get<string>("symbol");
+            var fc = new ContractUtils.FutureContract(contract);
+            return fc.future_tradable_contract != null;
+         });
+         r = new NonIncGroupByStream(r,
+                                      "ts_event",
+                                      "volume:Max");
+         r = new NewProjectStream(r,
+                                   "ts_event symbol !");
+         r = new SortStream(r,
+                             "-ts_event");
+         mySetRealSource(r);
+      }
+   }
 }
@@ -2,6 +2,93 @@ namespace bts.SilverPine {
   using bts;
   using BCG;
   using MongoDB.Driver;
+
+   // OHLCV request breakdown
+   // symbol_or_contract (if symbol, get continous, otherwise get raw and filter it)
+   // bar_size (one min, one day)
+   // live/historic
+   // time range
+
+   // internal code oranization only
+   public class OHLCV_1min_stream : ViewStream {
+
+      public OHLCV_1min_stream(string symbol_or_contract, string range_beg, string range_end) :
+      this(symbol_or_contract, DateTime.Parse(range_beg), DateTime.Parse(range_end)) { }
+
+      public OHLCV_1min_stream(string symbol_or_contract, DateTime range_beg, DateTime range_end) {
+         Func<string, bool, SQLiteStream> get_sqlite_str = (symbol, is_continous_contracts) => {
+            string table_name = is_continous_contracts ? $"ohlcv_1min_{symbol}_continous" : $"{symbol}_ohlcv_1min";
+            string db_str = $"Data Source={SQLiteHelper.location}{SQLiteHelper.db_name};Version=3";
+            string range_beg_iso8601 = range_beg.ToString("yyyy-MM-ddTHH:mm:ss"),
+               range_end_iso8601 = range_end.ToString("yyyy-MM-ddTHH:mm:ss");
+            string sql_str = $"select * from {table_name} where DATE(ts_event) >= '{range_beg_iso8601}' AND DATE(ts_event) <= '{range_end_iso8601}'";
+            return new SQLiteStream(db_str, sql_str);
+         };
+
+         Tuple<ContractUtils.ContractCategory, string> tup = ContractUtils.get_symbol_category(symbol_or_contract);
+         var contract_cat = tup.Item1;
+         switch (contract_cat) {
+            case ContractUtils.ContractCategory.Symbol: {
+                  ViewStream res = get_sqlite_str(tup.Item2, true);
+                  mySetRealSource(res);
+                  // this creates the continous stream for 1 min bars, but we store it in the DB, so we just read it from it
+                  // ViewStream ohlcv_1min = get_sqlite_str(tup.Item2, false);
+                  // // the following is whats written to the DB in the _contionous table
+                  // ohlcv_1min = new DateTimeExpanstionStream(ohlcv_1min, "ts_event");
+                  // ViewStream active_contr_str = new FuturesActiveContractStream(tup.Item2);
+                  // ViewStream res = new NonIncJoinStream(ohlcv_1min, "ts_event_date symbol", "ts_event ts_event_date symbol open high low close volume",
+                  //                                     active_contr_str, "ts_event contract", "ts_event:date contract:continous_contract");
+                  // res = new NullFilterStream(res, "continous_contract", true);
+                  // res = new NewProjectStream(res, "ts_event date continous_contract open high low close volume !");
+                  // mySetRealSource(res);
+                  break;
+               }
+            default: {
+                  ViewStream sql_str = get_sqlite_str(tup.Item2, false);
+                  ViewStream rs = new FilterStream(sql_str, $"symbol='{symbol_or_contract}'");
+                  // filter here
+                  mySetRealSource(rs);
+                  break;
+               }
+         }
+      }
+   }
+
+   // this is to make bigger bars form smaller bars. Right now the assumption it only works
+   // with symbols vs contracts, and the 1min <= aggregation_period < 1day
+   [DataStreamUserRespresentation("OHLCVAggregated")]
+   public class OHLCVAggregatedStream : ViewStream {
+      public OHLCVAggregatedStream(string symbol, string agregation_period, string range_beg, string range_end) :
+      this(symbol, TimeBasedFilter.parse_duration(agregation_period), DateTime.Parse(range_beg), DateTime.Parse(range_end)) {
+      }
+
+      public OHLCVAggregatedStream(string symbol, TimeSpan aggregation_period, DateTime range_beg, DateTime range_end) {
+         Tuple<ContractUtils.ContractCategory, string> tup = ContractUtils.get_symbol_category(symbol);
+         Log.assert(tup.Item1 == ContractUtils.ContractCategory.Symbol);
+         if (aggregation_period < TimeSpan.FromDays(1.0) &&
+            aggregation_period > TimeSpan.FromMinutes(1.0)) {
+            ViewStream r = new OHLCV_1min_stream(symbol, range_beg, range_end);
+            r = new DateTimeExpanstionStream(r,
+                                              "ts_event");
+            double tot_secs = aggregation_period.TotalSeconds;
+            r = new MathStream(r,
+                                $"bucket = Floor(ts_event_seconds_of_day / {tot_secs})");
+            r = new NonIncGroupByStream(r,
+                                         "bucket",
+                                         // column:operation
+                                         "open:first, high:max, low:min, close:last, volume:sum",
+                                         "ts_event");
+            r = new SortStream(r,
+                                "-ts_event");
+            mySetRealSource(r);
+         } else {
+            // fix it later
+            Log.assert(false);
+         }
+      }
+   }
+
+
   [DataStreamUserRespresentation("OHLCV")]
   // This stream retreives the OHLCV table as is, and fires it as-is. 
   // The only logic here is as follows:
@@ -80,12 +167,13 @@ namespace bts.SilverPine {
         switch (ohlcv_params.type) {
            case OHLCVType.LIVE:
            case OHLCVType.LIVE_AND_HISTORIC:
+               // TODO
               Log.assert(false);
               break;
            case OHLCVType.HISTORIC:
               // TODO: assuming 1 day only for historic               
               string table_name = get_table_from_barsize(ohlcv_params.bar_size, symbol);// $"{symbol}_OHLCV_1DAY";
-               //string table_name = $"{symbol}_{ohlcv_params.bar_size}";
+                                                                                         //string table_name = $"{symbol}_{ohlcv_params.bar_size}";
               string db_str = $"Data Source={_location}{_db_name};Version=3";
               string sql_str = $"select * from {table_name}";
               // "Data Source=C:\\Users\\m_tah\\development\\data\\stream_data.db;Version=3;"
@@ -111,6 +199,11 @@ namespace bts.SilverPine {
      :
      this(symbol, Enum.Parse<OHLCVBarSize>(bar_size)) { }

+      // aggregates the bars
+      public OHLCVStream(string symbol, TimeSpan agregation_period) {
+
+      }
+


   }
@@ -3,51 +3,24 @@ namespace bts.SilverPine {

   using bts;

-   using DnsClient.Protocol;
-
-   using MongoDB.Driver;
-
   [DataStreamUserRespresentation("Settlement")]
   public class SettlementPriceStream : ViewStream {

      public SettlementPriceStream(string future_name, string date_txt)
-      : this(future_name, date_txt, String.Empty) { }
+      : this(future_name, date_txt, date_txt) { }

      public SettlementPriceStream(string future_name, string begin_dt, string end_dt)
      : this(future_name, DateTime.Parse(begin_dt).Date, DateTime.Parse(end_dt).Date) { }

      public SettlementPriceStream(string symbol_or_contract, DateTime begin_dt)
-      : this(symbol_or_contract, begin_dt, DateTime.MinValue) { }
+      : this(symbol_or_contract, begin_dt, begin_dt) { }

      public SettlementPriceStream(string symbol_or_contract, DateTime begin_dt, DateTime end_dt) {
-         // DateTime begin_dt = DateTime.Parse(begin_dt_txt);
-         // DateTime end_dt = String.IsNullOrEmpty(end_dt_txt) ? DateTime.MinValue : DateTime.Parse(end_dt_txt);
-         Tuple<ContractUtils.ContractCategory, string> ccat_sym_contr = ContractUtils.get_symbol_category(symbol_or_contract);
-         ContractUtils.ContractCategory contr_cat = ccat_sym_contr.Item1;
-         string symbol = ccat_sym_contr.Item2;
-         ViewStream settl_str = new OHLCVStream(symbol);
-         // TODO: close != settlement, get settlement from somewhere
-         ViewStream rs = new NewProjectStream(settl_str, "close:settlement");
-         switch (contr_cat) {
-            case ContractUtils.ContractCategory.FutureString:
-            case ContractUtils.ContractCategory.SpreadString:
-               rs = new FilterStream(settl_str, r => r.get<string>("contract").CompareTo(symbol_or_contract) == 0);
-               break;
-            case ContractUtils.ContractCategory.Symbol:
-               rs = new FuturesActiveContractStream(settl_str);
-               break;
-         }
-         if (DateTime.MinValue.Equals(end_dt)) {
-            rs = new FilterStream(rs, r => r.get_datetime("ts_event").CompareTo(begin_dt) == 0);
-         }
-         rs = new FilterStream(rs, r => {
-            DateTime dt = r.get_datetime("ts_event");
-            return dt <= begin_dt && dt >= end_dt;
-         });
-         rs = new MathStream(rs, "settlement = close * 1.0");
-         rs = new NewProjectStream(rs, " ! ts_event:date settlement");
+         ContractUtils.FutureContract fc = new ContractUtils.FutureContract(symbol_or_contract);
+         ViewStream rs = new FutureHistoricalBarsStream(fc, BarSize.ONE_DAY, new DateTimeRange(begin_dt, end_dt));
+         rs = new NewProjectStream(rs, "symbol ts_event:date close:settlement !");
         mySetRealSource(rs);
      }

   }
-}
+}
@@ -0,0 +1,6 @@
+namespace bts.SilverPine;
+
+public class DatabaseConfig {
+   public string? Path { get; set; }
+   public string? Query { get; set; }
+}
@@ -41,70 +41,74 @@
      },
      {
         "category": "VWAP",
-         "description": "This is a few shot learning example for an NL to DSL. The DSL function in question is VWAP (symbol_or_contract, datetime1, datetime2). The first argument is a future (like CL, for crude oil, or ES for SP 500) or a specific future contract like crude oil Dec 24 future (CLZ4) or SP 500 Dec 2024 (ESZ4). The second argument is where the VWAP has to be calculated (also the beginning of the time window), and the third argument is datetime specifying the end of the time window. So, for example, a 20-day VWAP on 9/30/2023, the first argument will be 9/30/2023, and the third argument will be 9/10/2023 (20 days prior). This will also have datetimes expressed as Today, Yesterday, Last Friday, Last Sunday, etc. For the purpose of this few-shot prompts, let's assume today is 10/11/2024.",
+         "description": "This is a few shot learning example for an NL to DSL. The DSL function in question is VWAP (symbol_or_contract, perio_end_date, period_start_date). The first argument is a future (like CL, for crude oil, or ES for SP 500) or a specific future contract like crude oil Dec 24 future (CLZ4) or SP 500 Dec 2024 (ESZ4). The second and third argument define the end and start of the VWAP period, respectively. So for example, if we are calculating the 10 day VWAP on 10/15/2023, the period will be from 10/15/2023 going back to 10/05/2023. so the second argument will 10/05/2023 and the third argument will be 10/15/2023. The NL prompt  will also have datetimes expressed as Today, Yesterday, Last Friday, Last Sunday, First day of quarter, Today etc. You will need to translate these as dates that need to be passed to the VWAP DSL function. ",
         "prompt": "What was the VWAP for SP 500 Future over the past 30 days?",
         "examples": [
            {
               "natural_language": "Find the 20 day VWAP of the SP future on 04/22/2023.",
-               "dsl": "VWAP(\"ES\", 4/22/2023, 4/02/2023)"
+               "dsl": "VWAP(\"ES\", 4/02/2023, 4/22/2023)"
            },
            {
               "natural_language": "Find Yesterday's 10 day VWAP of the Crude Oil Future.",
-               "dsl": "VWAP(\"CL\", 10/10/2024, 10/1/2024)"
+               "dsl": "VWAP(\"CL\", 10/01/2024, 10/10/2024)"
            },
            {
               "natural_language": "Find today's 30-day VWAP of the 10 year treasury future.",
-               "dsl": "VWAP(\"ZN\", 10/11/2024, 9/11/2024)"
+               "dsl": "VWAP(\"ZN\", 9/11/2024, 10/11/2024)"
            },
            {
               "natural_language": "Find the 10 hour VWAP of the Natural Gas future from yesterday at 5:00 PM.",
-               "dsl": "VWAP(\"NG\", 9/27/2024 17:00:00, 9/27/2024 7:00:00)"
+               "dsl": "VWAP(\"NG\", 9/27/2024 7:00:00, 9/27/2024 17:00:00)"
            }
         ]
      },
      {
-         "category": "Price Diff",
-         "description": "This is a few shot learning example for an NL to DSL. The DSL function in question is OHLCVPriceDiff (symbol_or_contract, datetime1, datetime2). The first argument is a future (like CL, for crude oil) or a specific future contract like crude oil Dec 24 future (CLZ4). The second and third arguments are dates, that the function finds the difference of prices at. Assume today is 10/15/2024",
-         "prompt": "What is the difference in  SP 500 Future from July 10th to July 14th?",
+         "category": "FuturePriceDiff",
+         "description": "This is a few-shot learning example for an NL to DSL. The DSL function in question is FuturePriceDiff(symbol_or_contract, datetime1, datetime2). The first argument is a future (like CL, for crude oil, or ES for SP 500 and so on and so forth) or a specific future contract like crude oil Dec 24 future (CLZ4). The second and third arguments are datetimes (usually dates) that the function finds the difference of prices at. If the dates are given like \"first day of quarter\", \"first week of August\" I expect you to convert them into exact dates since the function only takes specific dates. The second argument should be the smaller datetime and the 3rd argument should be the end of the range, ie the higher one. Any prompt asking for price difference or price move should call FuturePriceDiff. If the dates you compute to pass to FuturePriceDiff are dates on which futures exchange might be closed (like 1/1, 12/25, Thanksgiving etc)  please adjust the date to the nearest viable trading date, for example if the date falls on christmas day, change it to 12/24 and so on and so on and so forth, or the date happens to be a saturday or sunday, change it to Friday or Monday depending on which is closest",
+         "prompt": "What is the difference in SP 500 Future from July 10th to July 14th?",
         "examples": [
            {
               "natural_language": "What is the difference in ZN price from July 10th to July 14th?",
-               "dsl": "OHLCVPriceDiff(\"ZN\", 7/14/2024, 7/10/2024)"
+               "dsl": "FuturePriceDiff(\"ZN\", 7/10/2024, 7/14/2024)"
            },
            {
               "natural_language": "What is the price difference in SP Future price from Sept 10th and Sept 5th?",
-               "dsl": "OHLCVPriceDiff(\"ES\", 9/10/2024, 9/5/2024)"
+               "dsl": "FuturePriceDiff(\"ES\", 9/5/2024, 9/10/2024)"
            },
            {
               "natural_language": "What is the price difference in Natural Gas Dec Future price from Aug 30th and Aug 20th?",
-               "dsl": "OHLCVPriceDiff(\"NGZ4\", 8/30/2024, 8/20/2024)"
+               "dsl": "FuturePriceDiff(\"NGZ4\", 8/20/2024, 8/30/2024)"
+            },
+            {
+               "natural_language": "What is the price move in  SP 500 Future from Aug 30th 2023 to October 10th 2023",
+               "dsl": "FuturePriceDiff(\"ES\", 8/30/2023, 10/10/2023)"
+            },
+            {
+               "natural_language": "What is the price move in  SP 500 Future during teh first quarter of last year",
+               "dsl": "FuturePriceDiff(\"ES\", 1/1/2023, 3/31/2023)"
            }
         ]
      },
      {
-         "category": "Absolute Volume",
-         "description": "this is a few shot learning example for an NL to DSL. The DSL function in question is AbsVolume(symbol, ContractType, datetime1, datetime2), which computes the highest volume in the date range, specified by datetime1, and datetime2. The first argument is a future symbol (like CL for crude oil, ES for SP 500 etc).  The second argument is the ContractType, which is \"Future\" for futures and \"FutureSpread\" for future spreads. the 3rd argument is the begin date and the 4th argument is the end date, between which the max volume is computed. If no future is specified use \"*\" from the first arugment. For this prompt, assume today is 10/11/2024 ",
+         "category": "Aggregate Volume",
+         "description": "this is a few shot learning example for an NL to DSL. The DSL function in question is VolumeAgg(symbol, contract_type, datetime1, datetime2), which computes the highest volume in the date range, specified by datetime1, and datetime2. The first argument is a future symbol (like CL for crude oil, ES for SP 500 etc).  The second argument is the ContractType, which is \"Future\" for futures and \"FutureSpread\" for future spreads. the 3rd argument is the begin date and the 4th argument is the end date, between which the max volume is computed. If no future is specified use \"*\" from the first arugment.",
         "prompt": "Which  SP 500 Future traded the most between 7/14/2024 and 7/10/2024",
         "examples": [
            {
-               "natural_language": "Which SP Future traded the most between 7/14/2024 and 7/10/2024",
-               "dsl": "AbsVolume(\"ES\", \"Future\", 7/14/2024, 7/10/2024)"
+               "natural_language": "Which SP Future traded the most between 7/10/2024 and 7/14/2024",
+               "dsl": "VolumeAgg(\"ES\", \"Future\", 1/1/2024, 1/7/2024)"
            },
            {
-               "natural_language": "Which Gold Spread traded the most between Today and Sept 20th",
-               "dsl": "AbsVolume(\"GC\", \"FutureSpread\", 10/1/2024, 9/20/2024)"
+               "natural_language": "Which Gold Spread traded the most in the first month of this year",
+               "dsl": "VolumeAgg(\"GC\", \"FutureSpread\", 1/1/2024, 1/31/2024)"
            },
            {
-               "natural_language": "Which 10 year Spread traded the highest volume last week?",
-               "dsl": "AbsVolume(\"ZN\", \"FutureSpread\", 9/27/2024, 9/23/2024)"
+               "natural_language": "Which 10 year Spread traded the highest volume last week to now?(in this example(only) we are assuming today is 11/26/2024)",
+               "dsl": "VolumeAgg(\"ZN\", \"FutureSpread\", 11/19/2024, 11/26/2024)"
            },
            {
-               "natural_language": "Which Corn Spread traded the most today?",
-               "dsl": "AbsVolume(\"ZC\", \"FutureSpread\", 10/1/2024, 10/1/2024)"
-            },
-            {
-               "natural_language": "Which Gold Future traded the most today?",
-               "dsl": "AbsVolume(\"GC\", \"Future\", 9/30/2024, 9/30/2024)"
+               "natural_language": "Which Corn Spread traded the most today (in this example(only) we are assuming today is 11/26/2024)?",
+               "dsl": "VolumeAgg(\"ZC\", \"FutureSpread\", 11/26/2024, 11/26/2024)"
            }
         ]
      },
@@ -218,6 +222,17 @@
               "dsl": "r=standardDeviation(\"ZN\", \"30d\")\nr=math(r, \"abs_daily_change = Abs(close - open) comp = std_dev * 2.0\")\nr=filter(r,\"abs_daily_change >= comp\")"
            }
         ]
+      },
+      {
+         "category": "Price Movement",
+         "description": "This is a few shot learning example for an NL to DSL. In the NL prompt the user is asking for the average price movement for e-minis (aka ES aka SP Futures aka SP 500 Futures)) on Wednesday between 9:30 am and 9:40 AM?  the multiline DSL is as follows (along with the explaination in comments before each line, dont include the comments in the asnwer):  //following line gets the 1 min ohlcv bars of ES \n r=OHLCV_1min_stream(\"ES\")\n //this passes it to the filter stream, which filters the rows for Wednesday, and time bw 9:30 and 9:40.\nr=filter(r,\"ts_event_day = 'Wednesday' and ts_event_hour = 9 and ts_event_min >= 30 and ts_event_min < 40\")\n// this aggregates it using the date as the agregation key, it find the max of the column called high, and the low of the column called min, to find the max and the min prices in that time window\nr=aggregate(r, \"ts_event_date\", \"high: max, low: min\")\n/// this then passes the previous result to the math stream operator which calculates the move, which is the difference between high_max and low_min (columns that were added from the previous aggregate)\nr=math(r,\"move = high_max - low_min\")\n// At the end we compute the mean of the move column, notice this is the total mean, so we are not providing an aggregation col to agregate\nr=aggregate(r, \"move:mean\")\n// we project out every col by the ! operator and call rename move_mean as average_vol\nr=newProject(r,\"move_mean:average_move !\")",
+         "prompt": "What is the average price movement for SP Futures (ES) on Wednesday between 9:30 am and 9:40 AM ?",
+         "examples": [
+            {
+               "natural_language": "What is the average price movement for SP Futures (ES) on Wednesday between 9:30 am and 9:40 AM?",
+               "dsl": "r=OHLCV_1min_stream(\"ES\")\nr=filter(r,\"ts_event_day = 'Wednesday' and ts_event_hour = 9 and ts_event_min >= 30 and ts_event_min < 40\")\nr=aggregate(r, \"ts_event_date\", \"high: max, low: min\")\nr=math(r,\"move = high_max - low_min\")\nr=aggregate(r, \"move:mean\")\nr=newProject(r,\"move_mean:average_move !\")"
+            }
+         ]
      }
   ]
 }
@@ -0,0 +1,77 @@
+namespace bts.SilverPine;
+
+using Spark.Core.Json;
+
+using System.Collections.Generic;
+using System.Text.Json;
+using System.Text.Json.Serialization;
+
+public class NLPromptConfig {
+   static NLPromptConfig? instance;
+
+   public static NLPromptConfig Instance =>
+      instance ?? throw new InvalidOperationException(
+         "SetGlobalConfig must be called to initialize config static singleton");
+
+   public List<FewShotPrompt> prompts { get; } = new(0);
+
+   public static void SetGlobalConfig(NLPromptConfig cfg) {
+      // This should only be called once
+      Log.isnull(instance);
+      instance = Log.notnull(cfg);
+   }
+
+   public static NLPromptConfig Default() => new();
+
+   // Represents a pair of a natural language query and its dsl
+   public class FewShotExample {
+      public string? natural_language { get; set; }
+      public string? dsl { get; set; }
+   }
+
+   public class FewShotPrompt {
+      public string? category { get; set; }
+
+      public string? description { get; set; }
+
+      public bool IsEnabled { get; set; } = true;
+
+      // MultiLineDescription is simply here for convenience. It is functionally redundant, as it
+      // serves the same purpose as 'description'. If 'description' is too long, you can break it
+      // up into an array of strings and assign it to MultiLineDescription instead.
+      public List<string>? MultiLineDescription { get; set; }
+
+      // the actual prompt given by the user/CME
+      public string? prompt { get; set; }
+
+      public List<FewShotExample> examples { get; } = new(0);
+
+      public override string ToString() {
+         var desc = description ?? string.Concat(MultiLineDescription.empty_if_null());
+         string rv = $"System: {desc}\n";
+         int ex_no = 0;
+         foreach (FewShotExample ex in examples ?? new(0)) {
+            rv += $"{++ex_no}. Natural Language: \"{ex.natural_language}\"" + "\n";
+            rv += $"    DSL: {ex.dsl}\n";
+
+         }
+         return rv;
+      }
+
+      // Reads normal strings and arrays of lines
+      public class FewShotPromptLinesConverter : JsonConverter<string> {
+         public override string? Read(ref Utf8JsonReader reader, Type typeToConvert, JsonSerializerOptions options) {
+            if (reader.ReadString(out var str)) {
+               return str;
+            }
+            if (reader.ReadArray(out var array)) {
+               return string.Join('\n', array.Select(v => v?.AsString()));
+            }
+            throw new JsonException("Failed to parse FewShotPrompt lines");
+         }
+
+         public override void Write(Utf8JsonWriter writer, string value, JsonSerializerOptions options) =>
+            writer.WriteStringValue(value);
+      }
+   }
+}
@@ -0,0 +1,42 @@
+namespace bts.SilverPine;
+
+using BCG;
+
+using Bts.DataStream;
+
+using mbts.utils;
+
+using System.Collections.Generic;
+
+public class NLStreamFactory : IRecursiveDataStreamFactory {
+   IDataStreamFactory? compositeFactory;
+
+   public ViewStream? Create(string streamId, IEnumerable<(string? Name, int? Index, object? Value)> args) {
+      if (SupportedStreams.NaturalLanguage.IsMatch(streamId)) {
+         if (compositeFactory is null) {
+            Log.DataFlow.warn("Failed to create NLStream because 'compositeFactory' is null.");
+            return null;
+         }
+         if (args.SingleOrDefault().Value is string prompt) {
+            return new NLStream(prompt, compositeFactory);
+         }
+      }
+      return null;
+   }
+
+   public IEnumerable<IDataStreamMetadata> GetSupportedStreams() {
+      yield return SupportedStreams.NaturalLanguage;
+   }
+
+   public void OnCompositeFactory(IDataStreamFactory compositeFactory) =>
+      this.compositeFactory = compositeFactory;
+
+   static class SupportedStreams {
+      public static readonly DataStreamMetadata NaturalLanguage = new() {
+         Name = "nl",
+         AlternateNames = new() { "naturalLanguage" }
+      };
+
+      public static string NlName => ExceptionAssert.not_null(NaturalLanguage.Name);
+   }
+}
@@ -40,4 +40,4 @@ namespace bts.SilverPine {


   }
-}
+}
@@ -7,38 +7,16 @@ namespace bts.SilverPine {

   using System.Text.Json;

+   using Root = NLPromptConfig;
+   using FewShotExample = NLPromptConfig.FewShotExample;
+   using FewShotPrompt = NLPromptConfig.FewShotPrompt;

   [DataStreamUserRespresentation("nl_to_indexed_fewshot_prompt")]
   // given an nl prompt, retreives the correct fewshot prompt and converts it into a form
   // that be sent to openAI as a prompt
   public class NLToIndexedFewShotPromptStream : ViewStream {

-      // Represents a pair of a natural language query and its dsl
-      public class FewShotExample {
-         public string natural_language { get; set; }
-         public string dsl { get; set; }
-      }
-
      // Class to represent each prompt category
-      public class FewShotPrompt {
-         public string category { get; set; }
-         // this is the explaination of the DSL
-         public string description { get; set; }
-         // th actual prompt given by the user/CME
-         public string prompt { get; set; }
-         public List<FewShotExample> examples { get; set; }
-
-         public override string ToString() {
-            string rv = $"System: {description}\n";
-            int ex_no = 0;
-            foreach (FewShotExample ex in examples) {
-               rv += $"{++ex_no}. Natural Language: \"{ex.natural_language}\"" + "\n";
-               rv += $"    DSL: {ex.dsl}\n";
-
-            }
-            return rv;
-         }
-      }

      // this is how it should be prepended to the actual prompt:
      //   System: This is a few-shot learning example for converting natural language prompts to a DSL function for settlement data retrieval. The DSL function can take either a single date or a date range. Assume today's date is 10/11/2024. The function is structured as follows:
@@ -66,21 +44,6 @@ namespace bts.SilverPine {

      // [Your Real Prompt Goes Here]

-
-      // Root class to map the JSON structure
-      public class Root {
-         public List<FewShotPrompt> prompts { get; set; }
-      }
-
-      public static Root read_json_file() {
-         // TODO
-         string json_file_path = @"C:\Users\m_tah\development\spark\src\DataStreamSuite\SharedStreams\NL\FewShotPrompts.json";
-         string json_string = File.ReadAllText(json_file_path);
-         var options = new JsonSerializerOptions { PropertyNameCaseInsensitive = true };
-         Root root = JsonSerializer.Deserialize<Root>(json_string, options);
-         return root;
-      }
-
      private readonly string _nl_prompt;
      private string _fewshot_index_prompt;
      private string _fewshot_index_reponse;
@@ -101,7 +64,7 @@ namespace bts.SilverPine {
      }

      protected override void myOnFirstSubscriber() {
-         Root root = read_json_file();
+         var root = NLPromptConfig.Instance;
         string meta_prompt = build_fewshot_index_prompt(root, _nl_prompt);
         _fewshot_index_prompt = meta_prompt;
         Log.Gen.info($"NLToDSLStream.myOnFirstSubscriber _fewshot_index_prompt: {_fewshot_index_prompt}");
@@ -138,7 +101,7 @@ namespace bts.SilverPine {
            string[] toks = response.Split('.');
            index = Int32.Parse(toks[0]);
         }
-         var root = read_json_file();
+         var root = NLPromptConfig.Instance;
         FewShotPrompt fs_prompt = root.prompts[index];
         string final_prompt = _build_fewshot_prompt(fs_prompt, _nl_prompt);
         _fire_event(final_prompt);
@@ -166,6 +129,10 @@ namespace bts.SilverPine {
         string meta_promp = "Take a look at the following prompt examples:\n";
         int rv = 0;
         foreach (FewShotPrompt fs_entry in root.prompts) {
+            if (!fs_entry.IsEnabled) {
+               continue;
+            }
+            var prompt = fs_entry.prompt;
            meta_promp += $"{rv++}. {fs_entry.prompt}\n";
         }
         meta_promp += "Based on the above prompts, tell me which prompt (give me answer as the index only, and nothing else) does the following prompt most closely matches to: \n";
@@ -194,4 +161,4 @@ namespace bts.SilverPine {
         return rv;
      }
   }
-}
+}
@@ -0,0 +1,32 @@
+namespace bts.SilverPine;
+
+// public class DateRange {
+//    public DateTime Begin { get; set; }
+//    public DateTime End { get; set; }
+// }
+
+public class OhlcvDatabaseConfig {
+   public class DatasetConfig {
+      public string? Path { get; set; }
+      public string? Query { get; set; }
+      public List<string>? QueryParams { get; set; }
+   }
+
+   public DatasetConfig? Bbo1s { get; set; }
+   public DatasetConfig? Bbo1m { get; set; }
+   public DatasetConfig? Ohlcv1s { get; set; }
+   public DatasetConfig? Ohlcv1m { get; set; }
+   public DatasetConfig? Ohlcv1h { get; set; }
+   public DatasetConfig? Ohlcv1d { get; set; }
+   public DatasetConfig? Mbo { get; set; }
+
+   // public List<string>? Symbols { get; set; }
+   // public List<string>? Venues { get; set; }
+   // public List<string>? Types { get; set; }
+   // public List<DateRange>? DateRanges { get; set; }
+   // public string? Path { get; set; }
+}
+
+public class OhlcvStreamConfig {
+   public OhlcvDatabaseConfig? Database { get; set; }
+}