Add scaffolding for PandasExpressions

Improves performance compared to accessing columns by name

.config/dotnet-tools.json

@@ -25,6 +25,12 @@
       "commands": [
         "fake"
       ]
+    },
+    "dotnet-t4": {
+      "version": "3.0.0",
+      "commands": [
+        "t4"
+      ]
     }
   }
-}
+}

.env

@@ -0,0 +1,81 @@
+# This file defines the global default configuration for docker deployments.
+# It is tracked in source control. For temporary modifications, please create
+# a separate file called ".env.local" (it's in .gitignore, so won't be tracked).
+# If a variable is defined in both files, the value in .env.local will be used.
+#
+# About the deployment procedure:
+#   We use Docker Compose to build images and launch containers for various services
+#   that make up our software suite:
+#     - sparkd
+#     - pubsub
+#     - md-publisher
+#     - webapp (DSQA on the web)
+#     - api-server (websocket based)
+#   From inside a devcontainer, you can launch all services in separate containers
+#   on your machine by running:
+#
+#     tools/deploy.sh
+#
+#   This script builds release versions for each service, packaged (generally) in
+#   self-contained directories. Next, it runs a few "docker compose" commands to
+#   build docker images loaded with the self-contained binaries from the previous
+#   step, and start containers with those images to run each service. To view the
+#   commands that deploy.sh runs internally (without actually running them), run:
+#
+#     tools/deploy.sh --dry-run
+#
+#   For additional information/help, run:
+#
+#     tools/deploy.sh --help
+#
+
+### Sparkd ###
+# The internal image for sparkd to use. For more info, view:
+#   src/app_sparkd/Dockerfile
+SPARKD__IMAGE=sparkd
+# The 'spark.conf' for this service to use
+SPARKD__CONF_FILE=src/app_sparkd/spark.conf
+# To run replay md, put recorded md files in SPARKD__MD_RECORDING_DIR, and use:
+#   SPARKD__IMAGE=sparkd_md_replay
+#   SPARKD__CONF_FILE=src/app_sparkd/spark.local.conf
+# (configure which md to replay in the new SPARKD__CONF_FILE)
+SPARKD__MD_RECORDING_DIR=.md_recording
+SPARKD__SOURCE_DIR=install/sparkd-release-gcc
+SPARKD__CONFIG_PORT=8000
+SPARKD__QUICKBUS_PORT=8001
+SPARKD__SIM_FIX_PORT=8002
+
+### Pubsub ###
+# The 'spark.conf' for this service to use
+PUBSUB__CONF_FILE=src/DataStreamSuite/PubSubServer/spark.conf
+PUBSUB__CS_SOURCE_DIR=build/Release/bin/net7.0/linux-x64/publish
+PUBSUB__CPP_SOURCE_DIR=install/data-stream-suite-linux
+# The appsettings to load at runtime. Expects a file named:
+#   src/DataStreamSuite/PubSub/appsettings.${PUBSUB__DOTNET_ENVIRONMENT}.json
+PUBSUB__DOTNET_ENVIRONMENT=Staging
+
+### Md Publisher ###
+# The 'spark.conf' for this service to use
+MD_PUBLISHER__CONF_FILE=src/DataStreamSuite/DataStreamPublisher/spark.conf
+MD_PUBLISHER__SOURCE_DIR=build/Release/bin/net7.0/linux-x64/publish
+# Determines which appsettings to load at runtime. Expects a file named:
+#   src/DataStreamSuite/DataStreamPublisher/appsettings.${MD_PUBLISHER__DOTNET_ENVIRONMENT}.json
+MD_PUBLISHER__DOTNET_ENVIRONMENT=MarketData
+
+### Webapp (DSQA on the web) ###
+# The 'spark.conf' for this service to use
+WEBAPP__CONF_FILE=src/client/spark_api_server/spark.conf
+WEBAPP__SOURCE_DIR=build/Release/bin/net7.0/linux-x64/publish
+WEBAPP__WEB_ROOT=build/Bts.DataStreamWebApp.Client/production/dist
+# Determines which appsettings to load at runtime. Expects a file named:
+#   src/DataStreamSuite/Bts.DataStreamWebApp.Server/appsettings.${WEBAPP__ASPNETCORE_ENVIRONMENT}.json
+WEBAPP__ASPNETCORE_ENVIRONMENT=Staging
+
+### Api Server ###
+# The 'spark.conf' for this service to use
+API_SERVER__CONF_FILE=src/client/spark_api_server/spark.conf
+API_SERVER__CS_SOURCE_DIR=build/Release/bin/net7.0/linux-x64/publish
+API_SERVER__CPP_SOURCE_DIR=install/spark-api-server-linux
+# Determines which appsettings to load at runtime. Expects a file named:
+#   src/client/spark_api_server/appsettings.${API_SERVER__ASPNETCORE_ENVIRONMENT}.json
+API_SERVER__ASPNETCORE_ENVIRONMENT=Staging

.gitignore

@@ -59,3 +59,6 @@ trace_data/
 .devcontainer/
 *.Generated.cs
 *.lst
+src/app_sparkd/spark.local.conf
+.env.local
+.md_recording

.vscode/launch.json

@@ -62,7 +62,7 @@
       "name": "Bts.DataStreamWebApp.Server",
       "type": "coreclr",
       "request": "launch",
-      "preLaunchTask": "Build DataStream WebApp (Debug)",
+      "preLaunchTask": "Build DataStream (Debug)",
       "program": "${workspaceFolder}/build/Debug/bin/net7.0/Bts.DataStreamWebApp.Server.dll",
       "cwd": "${workspaceFolder}/build/Debug/bin/net7.0",
       "env": {

cmake/InitLibrary.cmake

@@ -112,7 +112,7 @@ function(init_library)
   )
 
   # Add tests executable (if there are any)
-  if (${_name}_TEST_SOURCES)
+  if (${_name}_TEST_SOURCES AND NOT WIN32)
     add_executable(${_name}-tests ${${_name}_TEST_SOURCES})
 
     get_target_property(_lib_type ${_name} TYPE)

cmake/ProtobufGenerateCpp.cmake

@@ -15,7 +15,11 @@ function(protobuf_generate_cpp)
   cmake_parse_arguments(_ARGS "${_options}" "${_one_value_args}" "${_multi_value_args}" ${ARGN})
 
   if (NOT DEFINED _ARGS_PROTOC)
-    set(_ARGS_PROTOC "${EXTERNALS_BIN}/protoc")
+    if (WIN32)
+      set(_ARGS_PROTOC "${EXTERNALS_BIN}/protoc.exe")
+    else()
+      set(_ARGS_PROTOC "${EXTERNALS_BIN}/protoc")
+    endif()
   endif()
   set(${_ARGS_CXX_SOURCES})
   set(${_ARGS_CXX_HEADERS})

cmake/vs_filters.cmake

@@ -5,7 +5,7 @@ cmake_minimum_required(VERSION 3.5)
 cmake_policy(SET CMP0076 NEW)
 
 # add protos to the library to facilitate source_group
-target_sources(libspark PUBLIC ${Config_Protos} ${NonConfig_Protos} ${Service_Protos})
+target_sources(spark PUBLIC ${Config_Protos} ${NonConfig_Protos} ${Service_Protos})
 
 function(assign_source_group)
   foreach(_source IN ITEMS ${ARGN})

compose.yaml

@@ -3,52 +3,55 @@ services:
     build:
       dockerfile: src/app_sparkd/Dockerfile
       args:
-        - SOURCE_DIR=install/sparkd-release-gcc
-        - CONF_FILE=src/app_sparkd/spark.conf
+        - IMAGE=${SPARKD__IMAGE}
+        - SOURCE_DIR=${SPARKD__SOURCE_DIR}
+        - CONF_FILE=${SPARKD__CONF_FILE}
+        - MD_RECORDING_DIR=${SPARKD__MD_RECORDING_DIR}
     command: [ "./sparkd", "run" ]
     ports:
-      - 8000:8000
-      - 8001:8001
-      - 8002:8002
+      - "8000:8000"
+      - "8001:8001"
+      - "8002:8002"
 
   pubsub:
     environment:
-      - DOTNET_ENVIRONMENT=Staging
+      - DOTNET_ENVIRONMENT=${PUBSUB__DOTNET_ENVIRONMENT}
     build:
       dockerfile: src/DataStreamSuite/PubSubServer/Dockerfile
       args:
-        - SOURCE_DIR=build/Release/bin/net7.0/linux-x64/publish
-        - CONF_FILE=src/DataStreamSuite/PubSubServer/spark.conf
+        - CS_SOURCE_DIR=${PUBSUB__CS_SOURCE_DIR}
+        - CPP_SOURCE_DIR=${PUBSUB__CPP_SOURCE_DIR}
+        - CONF_FILE=${PUBSUB__CONF_FILE}
     ports:
-      - 22019:22019
+      - "22019:22019"
     command: [ "./PubSubServer" ]
 
   md-publisher:
     build:
       dockerfile: src/DataStreamSuite/DataStreamPublisher/Dockerfile
       args:
-        - SOURCE_DIR=build/Release/bin/net7.0/linux-x64/publish
-        - CONF_FILE=src/DataStreamSuite/DataStreamPublisher/spark.conf
+        - SOURCE_DIR=${MD_PUBLISHER__SOURCE_DIR}
+        - CONF_FILE=${MD_PUBLISHER__CONF_FILE}
     depends_on:
       - pubsub
       - sparkd
     command: [ "dockerize", "-wait", "tcp://pubsub:22019", "-wait", "tcp://sparkd:8000", "-timeout", "10s", "./DataStreamPublisher" ]
     environment:
-      - DOTNET_ENVIRONMENT=MarketData
+      - DOTNET_ENVIRONMENT=${MD_PUBLISHER__DOTNET_ENVIRONMENT}
 
   webapp:
     ports:
-      - 8080:8080
-      - 8443:8443
+      - "8080:8080"
+      - "8443:8443"
     build:
       dockerfile: src/DataStreamSuite/Bts.DataStreamWebApp.Server/Dockerfile
       args:
-        - SOURCE_DIR=build/Release/bin/net7.0/linux-x64/publish
-        - WEB_ROOT=build/Bts.DataStreamWebApp.Client/production/dist
-        - CONF_FILE=src/DataStreamSuite/Bts.DataStreamWebApp.Server/spark.conf
+        - SOURCE_DIR=${WEBAPP__SOURCE_DIR}
+        - WEB_ROOT=${WEBAPP__WEB_ROOT}
+        - CONF_FILE=${WEBAPP__CONF_FILE}
     command: [ "./Bts.DataStreamWebApp.Server" ]
     environment:
-      - ASPNETCORE_ENVIRONMENT=Staging
+      - ASPNETCORE_ENVIRONMENT=${WEBAPP__ASPNETCORE_ENVIRONMENT}
 
   api-server:
     depends_on:
@@ -60,7 +63,7 @@ services:
       dockerfile: src/client/spark_api_server/Dockerfile
     command: [ "dockerize", "-wait", "tcp://sparkd:8000", "-timeout", "10s", "./spark_api_server", "run" ]
     environment:
-      - ASPNETCORE_ENVIRONMENT=Staging
+      - ASPNETCORE_ENVIRONMENT=${API_SERVER__ASPNETCORE_ENVIRONMENT}
 
   sparkd-ping-test:
     depends_on:

src/DataStreamSuite/Bts.DataStreamWebApp.Client/src/DataTable.jsx

@@ -1,7 +1,7 @@
 import PropTypes from 'prop-types';
 import { useTheme } from '@mui/material/styles';
 import { DataGrid } from '@mui/x-data-grid';
-import './DataGrid.css'
+import './DataGrid.css';
 import { useMemo, useState } from 'react';
 
 export default function DataTable({ columns, rows }) {
@@ -23,7 +23,7 @@ export default function DataTable({ columns, rows }) {
         headerName: col.name,
         width: columnWidths[col.name],
       };
-    }
+    };
   }, [columnWidths]);
 
   return (

src/DataStreamSuite/Bts.DataStreamWebApp.Client/src/QueryStatusView.jsx

@@ -12,7 +12,7 @@ const QueryStatusView = ({ queryState, details, error }) => {
 
   let header = queryState;
   let headerContent = null;
-  let detailElements = [];
+  const detailElements = [];
   let severity = queryState == "Subscribed" ? "success" : "info";
   let cursor = details == null ? "default" : "pointer";
   if (error != null) {
@@ -20,14 +20,14 @@ const QueryStatusView = ({ queryState, details, error }) => {
     cursor = "pointer";
     header = `Error Code: ${error.code}`;
     headerContent = error.message;
-    detailElements.push(<Typography variant="body1">{error.data.error_message}</Typography>);
-    detailElements.push(<Typography variant="subtitle1">Stack Trace:</Typography>);
-    (error.data.stack_trace ?? []).forEach(line => detailElements.push(
-      <Typography variant="body2" component="pre" sx={{ whiteSpace: 'pre-wrap' }}>
+    detailElements.push(<Typography key="error-msg" variant="body1">{error.data.error_message}</Typography>);
+    detailElements.push(<Typography key="error-details" variant="subtitle1">Stack Trace:</Typography>);
+    (error.data.stack_trace ?? []).forEach((line, idx) => detailElements.push(
+      <Typography key={`stacktrace-${idx}`} variant="body2" component="pre" sx={{ whiteSpace: 'pre-wrap' }}>
         {line}
       </Typography>));
   } else if (details != null) {
-    detailElements.push(<Typography variant="body1">{details}</Typography>);
+    detailElements.push(<Typography key="status-msg" variant="body1">{details}</Typography>);
   }
   return (
     <Box sx={{ margin: theme.spacing(2, 0, 0, 0) }}>

src/DataStreamSuite/Bts.DataStreamWebApp.Client/src/ReactDataGrid.jsx

@@ -13,7 +13,7 @@ function getGridColDef(col) {
 export default function ReactDataGrid({columns, rows}) {
   return (
     <DataGrid className={"data-grid"} rows={rows} columns={columns.map(getGridColDef)} />
-  )
+  );
 }
 
 ReactDataGrid.propTypes = {

src/DataStreamSuite/Bts.DataStreamWebApp.Client/src/SideBar.jsx

@@ -3,7 +3,7 @@ import Button from '@mui/material/Button';
 import { useTheme } from '@mui/material/styles';
 import PropTypes from 'prop-types';
 import Tooltip from '@mui/material/Tooltip';
-import ChevronLeftRounded from "@mui/icons-material/ChevronLeftRounded"
+import ChevronLeftRounded from "@mui/icons-material/ChevronLeftRounded";
 import ChevronRightRounded from '@mui/icons-material/ChevronRightRounded';
 import FileCopyIcon from '@mui/icons-material/FileCopy';
 import './SideBar.css';
@@ -51,7 +51,7 @@ const Sidebar = ({ onSelection, onOpened, onClosed }) => {
       } catch (error) {
         console.error('Error fetching WebSocket config:', error);
       }
-    }
+    };
     fetchCannedQueries();
   }, []);
 

src/DataStreamSuite/Bts.DataStreamWebApp.Client/vite.config.js

@@ -51,11 +51,26 @@ export default defineConfig(({ $$command, mode, $$isSsrBuild, $$isPreview }) =>
       proxy: {
         '^/api': {
           target,
-          secure: false
+          secure: false,
+          configure: (proxy) => {
+            proxy.on('error', (err) => {
+              console.log('proxy error', err);
+            });
+            proxy.on('proxyReq', (proxyReq, req) => {
+              console.log(`Forwarding ${proxyReq.method} ${proxyReq.protocol}//${proxyReq.host}${proxyReq.path} to ${req.socket.remoteAddress}${req.url}`);
+            });
+            proxy.on('proxyRes', (proxyRes, req, res) => {
+              console.log(`Forwarding response ${req.socket.localAddress} to ${res.socket.remoteAddress}. Status=${proxyRes.statusCode}`);
+            });
+          },
         },
         '^/ws': {
           target,
           secure: false
+        },
+        '^/browse': {
+          target,
+          secure: false
         }
       },
       port: 5173,

src/DataStreamSuite/Bts.DataStreamWebApp.Server/Controllers/DataStreamSubscriptionController.cs

@@ -1,14 +1,21 @@
 namespace Bts.DataStreamWebApp.Server.Controllers;
 
+using Bts.DataStream;
+
 using Microsoft.AspNetCore.Mvc;
 
 [Route("/ws")]
 public class DataStreamSubscriptionController : ControllerBase {
    readonly ILogger logger;
+   readonly IDataStreamFactory streamFactory;
 
-   public DataStreamSubscriptionController(ILogger<DataStreamSubscriptionController> logger) {
+   public DataStreamSubscriptionController(ILogger<DataStreamSubscriptionController> logger, IEnumerable<IDataStreamFactory> streamFactories) {
       this.logger = logger;
       logger.LogInformation("Creating {ControllerType}", GetType().Name);
+      var streamFactoryList = streamFactories.ToList();
+      streamFactory = streamFactoryList.Count == 1
+         ? streamFactoryList[0]
+         : new CompositeDataStreamFactory(streamFactoryList);
    }
 
    [Route("/ws")]
@@ -21,7 +28,7 @@ public class DataStreamSubscriptionController : ControllerBase {
       try {
          using var webSocket = await HttpContext.WebSockets.AcceptWebSocketAsync();
          logger.LogInformation("Websocket connection established.");
-         using var server = new DataStreamWebSocketServer(webSocket, logger);
+         using var server = new DataStreamWebSocketServer(webSocket, streamFactory, logger);
          await server.Run(cancellationToken);
       } catch (Exception ex) {
          logger.LogError("DataStreamSubscriptionController.Run threw exception: {Exception}", ex);

src/DataStreamSuite/Bts.DataStreamWebApp.Server/DataStreamWebSocketServer.cs

@@ -2,6 +2,7 @@ namespace Bts.DataStreamWebApp.Server.Controllers;
 
 using bts.websocket_core.json_rpc;
 
+using Bts.DataStream;
 using Bts.DataStreamWebApp.Server.Messages;
 
 using System.Net.WebSockets;
@@ -13,9 +14,11 @@ public class DataStreamWebSocketServer : IDisposable, IAsyncDisposable {
    readonly ILogger logger;
    readonly WebSocket webSocket;
    readonly Dictionary<string, DataStreamSubscription> subscriptions = new();
+   readonly IDataStreamFactory streamFactory;
 
-   public DataStreamWebSocketServer(WebSocket webSocket, ILogger logger) {
+   public DataStreamWebSocketServer(WebSocket webSocket, IDataStreamFactory streamFactory, ILogger logger) {
       this.webSocket = webSocket;
+      this.streamFactory = streamFactory;
       this.logger = logger;
    }
 
@@ -67,7 +70,7 @@ public class DataStreamWebSocketServer : IDisposable, IAsyncDisposable {
       }
       var subId = subReq.SubscriptionId ?? throw new InvalidOperationException("SubscriptionId is null (parse false)");
       disposeSubscription(subId);
-      if (!subReq.ParseQuery(out var vs, out parseErr, out errCode)) {
+      if (!subReq.ParseQuery(streamFactory, out var vs, out parseErr, out errCode)) {
          logger.LogError("Invalid 'subscribe' request: {Reason}", parseErr);
          var rpcError = JsonRpcResponse.CreateError<SubscriptionError>(
             req.Id, new() { Code = (long)errCode, Message = "Failed to parse subscription request", Data = parseErr });

src/DataStreamSuite/Bts.DataStreamWebApp.Server/Messages/SubscribeRequest.cs

@@ -4,6 +4,7 @@ using BCG;
 
 using bts.websocket_core.json_rpc;
 
+using Bts.DataStream;
 using Bts.DataStreamExpressions;
 
 using System.Diagnostics.CodeAnalysis;
@@ -37,7 +38,12 @@ public class SubscribeRequest {
       return true;
    }
 
-   internal bool ParseQuery([NotNullWhen(true)] out ViewStream? stream, [NotNullWhen(false)] out SubscriptionError? error, out ErrorCode errorCode) {
+   internal bool ParseQuery(
+      IDataStreamFactory streamFactory,
+      [NotNullWhen(true)] out ViewStream? stream,
+      [NotNullWhen(false)] out SubscriptionError? error,
+      out ErrorCode errorCode
+   ) {
       stream = null;
       errorCode = ErrorCode.None;
       error = null;
@@ -54,7 +60,7 @@ public class SubscribeRequest {
       }
       Exception? exception = null;
       try {
-         stream = exp.getView(new Dictionary<string, StatementExpression>(0));
+         stream = exp.getView(new Dictionary<string, StatementExpression>(0), streamFactory);
       } catch (Exception ex) {
          exception = ex;
       }

src/DataStreamSuite/Bts.DataStreamWebApp.Server/Program.cs

@@ -1,5 +1,6 @@
 using Bts.DataStreamCore.Services;
 using Bts.PubSub.Services;
+using Bts.SharedStreams.Services;
 
 using Spark.Api.Services;
 using Spark.Core.Services;
@@ -78,5 +79,9 @@ static void ConfigureServices(IServiceCollection services, IConfiguration config
       .AddSparkApi(config.GetSection("SparkApi"))
       .AddMasterWorker(config.GetSection("MasterWorker"))
       .AddViewStreamDatabase()
+      .AddPrimitiveDataStreamFactory(config.GetSection("PrimitiveStreams"))
+      .AddOhlcvStreams(config.GetSection("StreamDatabase"))
+      .AddNaturalLanguageStreams(config.GetSection("NaturalLanguageStream"))
+      .AddCompositeDataStreamFactory()
       .AddPubSubClient(config.GetSection("PubSub"));
 }

src/DataStreamSuite/Bts.DataStreamWebApp.Server/appsettings.NaturalLanguageCustom.json

@@ -0,0 +1,45 @@
+{
+  "NaturalLanguagePrompt": {
+    "prompts": [
+      {
+        "category": "Impact of financial events on the market",
+        "description": "Show how trading metrics (e.g., price movement, volume) are affected by events related to finance (rate announcements, earnings reports, etc.).",
+        "prompt": "Show me the price movement and volume of Crude Oil futures during the 5-minute window following the last 5 OPEC meetings",
+        "examples": [
+          {
+            "natural_language": "Show me the price movement and volume of Crude Oil futures during the 5-minute window following the last 5 OPEC meetings",
+            "dsl": [
+              "time_idx=EventTimeIndex.OpecMeetings(which: \"last 5\", duration: \"+5min\")",
+              "contract_idx=ActiveFutureContracts(predicate: \"product==CL\")",
+              "Bars(time_index: time_idx, contract_index: contract_idx, fields: [\"price_movement\", \"volume\"])"
+            ]
+          },
+          {
+            "natural_language": "How do OPEC meetings affect the price of Crude futures? Examine the 10 minutes after each meeting this year.",
+            "dsl": [
+              "time_idx=EventTimeIndex.OpecMeetings(duration: \"+10min\", predicate: \"timestamp.year==2024\")",
+              "contract_idx=ActiveFutureContracts(predicate: \"product==CL\")",
+              "Bars(time_index: time_idx, contract_index: contract_idx, fields: [\"price_movement\", \"volume\"])"
+            ]
+          },
+          {
+            "natural_language": "Show me how the SP 500 price moves after FOMC rate announcements that day",
+            "dsl": [
+              "time_idx=EventTimeIndex.FomcRateAnnouncements(which: \"timestamp.year==2024\", duration: \"+10min\")",
+              "contract_idx=ActiveFutureContracts(predicate: \"product==ES\")",
+              "Bars(time_index: time_idx, contract_index: contract_idx, fields: [\"price_movement\", \"volume\"])"
+            ]
+          },
+          {
+            "natural_language": "Analyze the impact of the FED's interest rate hikes in 2022 on the 10-year Treasury yield, focusing on the 2-week period following each hike.",
+            "dsl": [
+              "time_idx=FedInterestRateAnnouncements(predicate: \"timestamp.year==2022 && change_pct>0\", duration: \"+2 weeks\")",
+              "contract_idx=ActiveFutureContracts(predicate: \"product==ZN\")",
+              "Bars(time_index: time_idx, contract_index: contract_idx)"
+            ]
+          }
+        ]
+      }
+    ]
+  }
+}

src/DataStreamSuite/Bts.DataStreamWebApp.Server/appsettings.json

@@ -11,8 +11,14 @@
       "Http": {
         "Url": "http://0.0.0.0:8080"
       },
+      "HttpIpv6": {
+        "Url": "http://[::1]:8080"
+      },
       "Https": {
         "Url": "https://0.0.0.0:8443"
+      },
+      "HttpsIpv6": {
+        "Url": "https://[::1]:8443"
       }
     }
   },
@@ -22,6 +28,358 @@
   "ViewStreamDatabase": {
     "Interval": "00:00:00.1"
   },
+  "PrimitiveStream": {
+    "Assemblies": [
+      "DataStream.dll",
+      "SharedStreams.dll"
+    ],
+    "ExcludedTypes": [
+      "SettlementPriceStream",
+      "OHLCVStream"
+    ]
+  },
+  "StreamDatabase": {
+    "Path": "/home/bts/.local/share/bts/silver_pine/store.db",
+    "Query": "select * from metadata"
+  },
+  "NaturalLanguageStream": {
+    "prompts": [
+      {
+        "category": "Settlement",
+        "MulitLineDescription": [
+          "This is a few shot learning example for an NL to DSL. The DSL function in question is Settlement (symbol_or_contract, datetime1). ",
+          "The first argument is a future (like CL, for crude oil) or a specific future contract like crude oil Dec 24 future (CLZ4). ",
+          "The second argument is the date at which the settlement is to be fetched. There is an overloaded function, Settlement ",
+          "(symbol_or_contract, begin_dt, end_dt) which gets the settlements for all dates from begin_dt to end_dt. This will also have ",
+          "datetimes expressed as Today, Yesterday, Last Friday, First Friday of the month, Last Thursday of August etc. You will need to know ",
+          "what Today is, since there will be referecnes to last week, yesterday etc. These examples are assuming Today is 10/11/2024, however ",
+          "you should use the Today's real date"
+        ],
+        "prompt": "What was the settlement for SP 500 Future on June 8th?",
+        "examples": [
+          {
+            "natural_language": "Find Settlement price of the SP500 future on 04/22/2023.",
+            "dsl": "Settlement(\"ES\", 4/22/2023)"
+          },
+          {
+            "natural_language": "Find Settlement price of the Silver Dec 24 Future on 04/22/2023.",
+            "dsl": "Settlement(\"SIZ4\", 4/22/2023)"
+          },
+          {
+            "natural_language": "Find Yesterday's Settlement of the Crude Oil Future.",
+            "dsl": "Settlement(\"CL\", 10/10/2024)"
+          },
+          {
+            "natural_language": "Find today's settlement of the 10 year treasury future.",
+            "dsl": "Settlement(\"ZN\", 10/11/2024)"
+          },
+          {
+            "natural_language": "Find the Settlement of Natural Gas Dec 24 future on the first Friday of Sept.",
+            "dsl": "Settlement(\"NGZ4\", 9/6/2024)"
+          },
+          {
+            "natural_language": "Find the Settlement of WTI Jan 25 future in the first Week of Sept.",
+            "dsl": "Settlement(\"CLF5\", 9/2/2024, 9/6/2024)"
+          },
+          {
+            "natural_language": "Find the Settlement price for the Sp 500 Dec 24 Mar 25  spread for the first two weeks of September.",
+            "dsl": "Settlement(\"ESZ4-ESH5\", 9/14/2024, 9/1/2024)"
+          },
+          {
+            "natural_language": "Find the Settlement of Natural Gas future on the first Friday of Sept.",
+            "dsl": "Settlement(\"NG\", 9/6/2024)"
+          }
+        ]
+      },
+      {
+        "category": "BBO",
+        "MulitLineDescription": [
+          "This is a few shot learning example for an NL to DSL. The DSL function in question is Bbo(args...). ",
+          "It accepts the following parameters: ",
+          "symbol (a product code such as ZN for 10 year T-Note futures), ",
+          "contract (a specific contract, such as ZNZ4 for the 10 Year T-Note future Dec 2024 exp), ",
+          "venue (name of the exchange, such as CME, NASDAQ, NYSE, NYMEX, DERIBIT etc.), ",
+          "interval (sample frequency of the data), ",
+          "begin and end (inclusive beginning and end of the time window to query, respectively), ",
+          "Arguments are specified by name, for example: Bbo(symbol: \"ZN\", venue: \"CME\", interval: \"PT1M\", begin: \"2024-10-10\", end: \"2024-10-11\"). ",
+          "Depending on the wording of the request, some arguments may be omitted. ",
+          "The begin, end, and interval args should be expressed in iso 8601 format. ",
+          "Interpret dates according to the trading schedule rather than calendar date whenever possible. ",
+          "In some prompts, the time window will be expressed in relative terms (e.g., \"last Monday\", or \"yesterday\") ",
+          "which must be translated to exact dates. To correctly interpret the dsl in the examples, assume the current time is 2024-10-11T12:00-05:00."
+        ],
+        "prompt": "Show me bbo for SP 500 futures on Sep 23rd of this year.",
+        "examples": [
+          {
+            "natural_language": "Get SP500 future bbo from Wednesday.",
+            "dsl": "Bbo(symbol: \"ES\", venue: \"CME\", begin: \"2024-10-08T17:00:00-06:00\", end: \"2024-10-09T16:00:00-06:00\")"
+          },
+          {
+            "natural_language": "Get bbo for 10 year T-Notes in the first minute after the market opened yesterday.",
+            "dsl": "Bbo(symbol: \"ES\", venue: \"CME\", begin: \"2024-10-10T17:00:00-06:00\", end: \"2024-10-10T17:01:00-06:00\")"
+          },
+          {
+            "natural_language": "Retrieve NVIDIA L1 books from Oct 3",
+            "dsl": "Bbo(symbol: \"NVDA\", venue: \"NASDAQ\", begin: \"2024-10-02TT18:00:00-05:00\", end: \"2024-10-03T17:00:00-05:00\")"
+          },
+          {
+            "natural_language": "Show me L1 data for Crude oil futures this week",
+            "dsl": "Bbo(symbol: \"CL\", venue: \"CME\", begin: \"2024-10-06:T17:00:00-06:00\")"
+          },
+          {
+            "natural_language": "Fetch hourly bbo for Gold futures this week",
+            "dsl": "Bbo(symbol: \"GC\", venue: \"CME\", begin: \"2024-10-06:T17:00:00-06:00\", interval: \"PT1H\")"
+          },
+          {
+            "natural_language": "What was the minute by minute bbo for Tesla yesterday?",
+            "dsl": "Bbo(symbol: \"TSLA\", venue: \"NASDAQ\", begin: \"2024-10-09T18:00:00-05:00\", end: \"2024-10-10T17:00:00-05:00\", interval: \"PT1M\")"
+          }
+        ]
+      },
+      {
+        "category": "Order flow imbalance",
+        "MulitLineDescription": [
+          "This is a few shot learning example for an NL to DSL. The DSL function in question is OrderImbalance(args...). ",
+          "It accepts the following parameters: ",
+          "symbol (a product code such as ZN for 10 year T-Note futures), ",
+          "contract (a specific contract, such as ZNZ4 for the 10 Year T-Note future Dec 2024 exp), ",
+          "venue (name of the exchange, such as CME, NASDAQ, NYSE, NYMEX, DERIBIT etc.), ",
+          "interval (sample frequency of the data), ",
+          "levels (the number of price levels to examine)",
+          "begin and end (inclusive beginning and end of the time window to query, respectively), ",
+          "Arguments are specified by name, for example: OrderImbalance(symbol: \"ZN\", venue: \"CME\", interval: \"PT1M\", begin: \"2024-10-10\", end: \"2024-10-11\"). ",
+          "Depending on the wording of the request, some arguments may be omitted. ",
+          "The begin, end, and interval args should be expressed in iso 8601 format. ",
+          "If the prompt doesn't contain an explicit interval, please choose a reasonable value depending on the duration between begin and end.",
+          "Interpret dates according to the trading schedule rather than calendar date whenever possible. ",
+          "In some prompts, the time window will be expressed in relative terms (e.g., \"last Monday\", or \"yesterday\") ",
+          "which must be translated to exact dates. To correctly interpret the dsl in the examples, assume the current time is 2024-10-11T12:00-05:00."
+        ],
+        "examples": [
+          {
+            "natural_language": "Get SP500 order imbalance from Wednesday.",
+            "dsl": "OrderImbalance(symbol: \"ES\", venue: \"CME\", begin: \"2024-10-08T17:00:00-06:00\", end: \"2024-10-09T16:00:00-06:00\", \"PT1M\")"
+          },
+          {
+            "natural_language": "Get SP500 order imbalance from Wednesday for the top 5 price levels.",
+            "dsl": "OrderImbalance(symbol: \"ES\", venue: \"CME\", begin: \"2024-10-08T17:00:00-06:00\", end: \"2024-10-09T16:00:00-06:00\", levels: 5, interval: \"PT1M\")"
+          },
+          {
+            "natural_language": "Get orderflow imbalance for 10 year T-Notes in the first minute after the market opened yesterday.",
+            "dsl": "OrderImbalance(symbol: \"ES\", venue: \"CME\", begin: \"2024-10-10T17:00:00-06:00\", end: \"2024-10-10T17:01:00-06:00\", interval: \"PT1S\")"
+          },
+          {
+            "natural_language": "Retrieve NVIDIA order flow from Oct 3",
+            "dsl": "OrderImbalance(symbol: \"NVDA\", venue: \"NASDAQ\", begin: \"2024-10-02TT18:00:00-05:00\", end: \"2024-10-03T17:00:00-05:00\", interval: \"PT1S\")"
+          },
+          {
+            "natural_language": "Show me order imbalance for Crude oil futures this week",
+            "dsl": "OrderImbalance(symbol: \"CL\", venue: \"CME\", begin: \"2024-10-06:T17:00:00-06:00\", interval: \"PT1H\")"
+          },
+          {
+            "natural_language": "Fetch hourly orderflow for Gold futures this week",
+            "dsl": "OrderImbalance(symbol: \"GC\", venue: \"CME\", begin: \"2024-10-06:T17:00:00-06:00\", interval: \"PT1H\")"
+          },
+          {
+            "natural_language": "What was the minute by minute orderflow for Tesla yesterday?",
+            "dsl": "OrderImbalance(symbol: \"TSLA\", venue: \"NASDAQ\", begin: \"2024-10-09T18:00:00-05:00\", end: \"2024-10-10T17:00:00-05:00\", interval: \"PT1M\")"
+          }
+        ]
+      },
+      {
+        "category": "VWAP",
+        "description": "This is a few shot learning example for an NL to DSL. The DSL function in question is VWAP (symbol_or_contract, datetime1, datetime2). The first argument is a future (like CL, for crude oil, or ES for SP 500) or a specific future contract like crude oil Dec 24 future (CLZ4) or SP 500 Dec 2024 (ESZ4). The second argument is where the VWAP has to be calculated (also the beginning of the time window), and the third argument is datetime specifying the end of the time window. So, for example, a 20-day VWAP on 9/30/2023, the first argument will be 9/30/2023, and the third argument will be 9/10/2023 (20 days prior). This will also have datetimes expressed as Today, Yesterday, Last Friday, Last Sunday, etc. For the purpose of this few-shot prompts, let's assume today is 10/11/2024.",
+        "prompt": "What was the VWAP for SP 500 Future over the past 30 days?",
+        "examples": [
+          {
+            "natural_language": "Find the 20 day VWAP of the SP future on 04/22/2023.",
+            "dsl": "VWAP(\"ES\", 4/22/2023, 4/02/2023)"
+          },
+          {
+            "natural_language": "Find Yesterday's 10 day VWAP of the Crude Oil Future.",
+            "dsl": "VWAP(\"CL\", 10/10/2024, 10/1/2024)"
+          },
+          {
+            "natural_language": "Find today's 30-day VWAP of the 10 year treasury future.",
+            "dsl": "VWAP(\"ZN\", 10/11/2024, 9/11/2024)"
+          },
+          {
+            "natural_language": "Find the 10 hour VWAP of the Natural Gas future from yesterday at 5:00 PM.",
+            "dsl": "VWAP(\"NG\", 9/27/2024 17:00:00, 9/27/2024 7:00:00)"
+          }
+        ]
+      },
+      {
+        "category": "Price Diff",
+        "description": "This is a few shot learning example for an NL to DSL. The DSL function in question is OHLCVPriceDiff (symbol_or_contract, datetime1, datetime2). The first argument is a future (like CL, for crude oil) or a specific future contract like crude oil Dec 24 future (CLZ4). The second and third arguments are dates, that the function finds the difference of prices at. Assume today is 10/15/2024",
+        "prompt": "What is the difference in  SP 500 Future from July 10th to July 14th?",
+        "examples": [
+          {
+            "natural_language": "What is the difference in ZN price from July 10th to July 14th?",
+            "dsl": "OHLCVPriceDiff(\"ZN\", 7/14/2024, 7/10/2024)"
+          },
+          {
+            "natural_language": "What is the price difference in SP Future price from Sept 10th and Sept 5th?",
+            "dsl": "OHLCVPriceDiff(\"ES\", 9/10/2024, 9/5/2024)"
+          },
+          {
+            "natural_language": "What is the price difference in Natural Gas Dec Future price from Aug 30th and Aug 20th?",
+            "dsl": "OHLCVPriceDiff(\"NGZ4\", 8/30/2024, 8/20/2024)"
+          }
+        ]
+      },
+      {
+        "category": "Absolute Volume",
+        "description": "this is a few shot learning example for an NL to DSL. The DSL function in question is AbsVolume(symbol, ContractType, datetime1, datetime2), which computes the highest volume in the date range, specified by datetime1, and datetime2. The first argument is a future symbol (like CL for crude oil, ES for SP 500 etc).  The second argument is the ContractType, which is \"Future\" for futures and \"FutureSpread\" for future spreads. the 3rd argument is the begin date and the 4th argument is the end date, between which the max volume is computed. If no future is specified use \"*\" from the first arugment. For this prompt, assume today is 10/11/2024 ",
+        "prompt": "Which  SP 500 Future traded the most between 7/14/2024 and 7/10/2024",
+        "examples": [
+          {
+            "natural_language": "Which SP Future traded the most between 7/14/2024 and 7/10/2024",
+            "dsl": "AbsVolume(\"ES\", \"Future\", 7/14/2024, 7/10/2024)"
+          },
+          {
+            "natural_language": "Which Gold Spread traded the most between Today and Sept 20th",
+            "dsl": "AbsVolume(\"GC\", \"FutureSpread\", 10/1/2024, 9/20/2024)"
+          },
+          {
+            "natural_language": "Which 10 year Spread traded the highest volume last week?",
+            "dsl": "AbsVolume(\"ZN\", \"FutureSpread\", 9/27/2024, 9/23/2024)"
+          },
+          {
+            "natural_language": "Which Corn Spread traded the most today?",
+            "dsl": "AbsVolume(\"ZC\", \"FutureSpread\", 10/1/2024, 10/1/2024)"
+          },
+          {
+            "natural_language": "Which Gold Future traded the most today?",
+            "dsl": "AbsVolume(\"GC\", \"Future\", 9/30/2024, 9/30/2024)"
+          }
+        ]
+      },
+      {
+        "category": "Roll",
+        "description": "This is a few shot learning example for an NL to DSL. The DSL function in question is FuturesRoll(symbol) which computes the rollover date for a future, ie when the highest volume swtches from one expiration to the next one. ",
+        "prompt": "When does the roll typically occur In  SP 500?",
+        "examples": [
+          {
+            "natural_language": "When does the roll typically occur In CL?",
+            "dsl": "FuturesRoll(\"CL\")"
+          },
+          {
+            "natural_language": "When does the roll typically occur for 10 year treasury?",
+            "dsl": "FuturesRoll(\"ZN\")"
+          },
+          {
+            "natural_language": "When does the roll usually occur for Gold Future",
+            "dsl": "FuturesRoll(\"GC\")"
+          },
+          {
+            "natural_language": "At what date does the future rollover happens for SP500?",
+            "dsl": "FuturesRoll(\"ES\")"
+          },
+          {
+            "natural_language": "When does the future rollover happens for the Corn Future?",
+            "dsl": "FuturesRoll(\"ZC\")"
+          }
+        ]
+      },
+      {
+        "category": "Daily Move",
+        "description": "This is a few shot learning example for an NL to DSL. ",
+        "prompt": "How often did Sp 500 Future settle + or - 10% over the previous day?",
+        "examples": [
+          {
+            "natural_language": "List all the days when the SP 500 closed + or -5% from the previous day's close",
+            "dsl": "ColDiff(filteredOHLCV(\"ES\"),\"ts_event\", -1, \"close\", \"close\", \"close_diff\", 5)"
+          },
+          {
+            "natural_language": "List all the days when the Crude Oil closed + or -2.5% from the previous day's close",
+            "dsl": "ColDiff(filteredOHLCV(\"CL\"),\"ts_event\", -1, \"close\", \"close\", \"close_diff\", 2.5)"
+          },
+          {
+            "natural_language": "Show me all the days when the Ten year Treaury Future opened + or -1% from the previous day's open",
+            "dsl": "ColDiff(filteredOHLCV(\"ZN\"),\"ts_event\", -1, \"open\", \"open\", \"open_diff\", 1.0)"
+          },
+          {
+            "natural_language": "List all the trading days when the Silver Future opened + or -1% from the previous day's close",
+            "dsl": "ColDiff(filteredOHLCV(\"SI\"),\"ts_event\", -1, \"open\", \"open\", \"open_diff\", 1.0)"
+          }
+        ]
+      },
+      {
+        "category": "Average Volume",
+        "description": "This is a few shot learning example for an NL to DSL. For the purpose of this few shot prompts, lets assume today is 10/3/2024 ",
+        "prompt": "When does the most open interest typically shift from one contract to another in SP 500 Future?",
+        "examples": [
+          {
+            "natural_language": "When does the most open interest typically shift from one contract to another in e-minis? ",
+            "dsl": "FuturesRoll(\"ES\")"
+          },
+          {
+            "natural_language": "Usually when does the open interest move from one contract to another in WTI?",
+            "dsl": "FuturesRoll(\"CL\")"
+          }
+        ]
+      },
+      {
+        "category": "Events",
+        "description": "This is a few shot learning example for an NL to DSL. In the NL prompt the user is asking to show the price of a future (treasury future, or SP 500 future etc) around an event type. The event type could be \"FOMC Announcement\" or \"Storage numbers\" etc. The DSL is as follows: EventsPriceAction(future_name, event_type, pre_event_window, post_event_window, number_of_events) first arg is the name of the future, (ex ES or ZN etc), the second argument is the event type, for example, \"FOMC announcement\", the 3rd argument is a time window for how much pre-event data to include in the end result, the 4th argument is how much post-event data to include in the end result. Both are strings of the form \"5d\" for 5 days, \"5h\" for 5 hours and \"120m\" for 120 minutes and so on. Usually, only one of the event periods is provided. Set the other to be half of what the user provided. The last argument is the number of events to consider. So if it’s 7, that means the most recent 7 events. If the user does not specify that, pass a large number.",
+        "prompt": "What did the volume and price of SP500 Future do following the release of the last 10 FOMC rates?",
+        "examples": [
+          {
+            "natural_language": "How did SP 500 Future do in the 5 days following the FOMC announcements?",
+            "dsl": "EventsPriceAction(\"ES\", \"FOMC Announcement\", \"2d\", \"5d\", 10)"
+          },
+          {
+            "natural_language": "How did the price of Treasury Future evolve in the 2 days following the Job Reports?",
+            "dsl": "EventsPriceAction(\"ZN\", \"Job Reports\", \"1d\", \"2d\", 10)"
+          },
+          {
+            "natural_language": "Show me the price movement and volume of Crude Oil futures during the 5-minute window following the last 5 OPEC meetings.",
+            "dsl": "EventsPriceAction(\"CL\", \"OPEC Meeting\", \"2m\", \"5m\", 5)"
+          },
+          {
+            "natural_language": "Show me the price movement and volume of SP 500 futures preceding the 10-minute window of the last 5 OPEC meetings.",
+            "dsl": "EventsPriceAction(\"CL\", \"OPEC Meeting\", \"10m\", \"5m\", 5)"
+          }
+        ]
+      },
+      {
+        "category": "Indicator, Standard Deviation",
+        "MulitLineDescription": [
+          "This is a few shot learning example for an NL to DSL. In the NL prompt the user is asking to show all the days when ",
+          "the price of a future (treasury future, or SP 500 future etc) moved more than m times the N days standard deviation, ",
+          "where both m and N are provided. For example, the DSL for \"Using a 90d standard deviation, give me the dates and ",
+          "amounts of the last 4+ standard deviation moves in WTI futures\" would be: ",
+          "r=standardDeviation(\"ES\", \"90d\")\n",
+          "r=math(r, \"abs_daily_change = Abs(close - open) comp = std_dev * 4\")\n",
+          "r=filter(r,\"abs_daily_change >= comp\"). ",
+          "The first line, creates a stream for StandardDeviation, based on the number of days, for example 90d standard deviation, ",
+          "and assigned it to r, the second line takes r from the first line and passes it to a math stream that creates 2 new columns, ",
+          "abs_daily_change and comp. comp is the factor (4, from 4+ in the prompt) times standard deviation. The third line filters ",
+          "(like a sql where) of rows where abs_daily_change >= comp"
+        ],
+        "prompt": "Using a 90d standard deviation, give me the dates and amounts of the last 1+ standard deviation moves in SP  futures ?",
+        "examples": [
+          {
+            "natural_language": "Using a 90d standard deviation, give me the dates and amounts of the last 1+ standard deviation moves in WTI futures ?",
+            "dsl": "r=standardDeviation(\"CL\", \"90d\")\nr=math(r, \"abs_daily_change = Abs(close - open) comp = std_dev * 1.0\")\nr=filter(r,\"abs_daily_change >= comp\")"
+          },
+          {
+            "natural_language": "Show me the times the SP Futures moved twice the 60 day standard deviation",
+            "dsl": "r=standardDeviation(\"ES\", \"60d\")\nr=math(r, \"abs_daily_change = Abs(close - open) comp = std_dev * 2.0\")\nr=filter(r,\"abs_daily_change >= comp\")"
+          },
+          {
+            "natural_language": "List all the days the Gold Futures moved 1.5 times the 45 day standard deviation",
+            "dsl": "r=standardDeviation(\"GC\", \"45d\")\nr=math(r, \"abs_daily_change = Abs(close - open) comp = std_dev * 1.5\")\nr=filter(r,\"abs_daily_change >= comp\")"
+          },
+          {
+            "natural_language": "List all the days the 10 year treasury future moved more than twice the 1 month standard deviation",
+            "dsl": "r=standardDeviation(\"ZN\", \"30d\")\nr=math(r, \"abs_daily_change = Abs(close - open) comp = std_dev * 2.0\")\nr=filter(r,\"abs_daily_change >= comp\")"
+          }
+        ]
+      }
+    ]
+  },
   "SparkApi": {
     "IsEnabled": false,
     "Credentials": {
@@ -77,7 +435,7 @@
         {
           "TagType": ".*",
           "Assembly": "DataStreamCore",
-         "Class": "bts.PubsubServerGenericDataHandler"
+          "Class": "bts.PubsubServerGenericDataHandler"
         }
       ]
     }

src/DataStreamSuite/DataStream/DataStreamExtensions.cs

@@ -2,17 +2,26 @@ namespace Bts.DataStream;
 
 using BCG;
 
+using bts.api.core;
+
+using Bts.DataStreamCore;
+
+using Microsoft.Extensions.Logging;
+
 using Spark.Core.Disposables;
 
 using System;
+using System.Collections;
+using System.Collections.Generic;
+using System.Linq;
 using System.Reactive.Linq;
 
 public static class DataStreamExtensions {
    public static ViewStream AsViewStream(this IObservable<DataPacket> source) =>
       source as ViewStream ?? new ObservableViewStream(source);
 
-   public static ViewStream AsViewStream<T>(this IObservable<IDataPacket<T>> source) where T : IBlueRow =>
-      source as ViewStream ?? new ObservableViewStream(source.Select(dp => new DataPacket(dp.newRows, dp.addedRows, dp.deletedRows)));
+   public static ViewStream AsViewStream<T>(this IObservable<IDataPacket<T>> source, string? name = null, ILogger? logger = null) where T : IBlueRow =>
+      source as ViewStream ?? new ObservableViewStream(source.Select(dp => new DataPacket(dp.newRows, dp.addedRows, dp.deletedRows)), name, logger);
 
    public static IObservable<DataPacket> AsObservable(this ViewStream source) {
       return source as IObservable<DataPacket> ?? Observable.Create<DataPacket>(Subscribe);
@@ -26,4 +35,100 @@ public static class DataStreamExtensions {
                $"Expected DataPacket, got {newValue?.GetType()}", nameof(newValue)));
       }
    }
+
+   public static IObservable<IDataPacket<T>> SelectDataPacket<T>(this IObservable<IDataPacket> source, Func<IBlueRow, T> selector) where T : IBlueRow =>
+      source.Select(dp => {
+         dp.get_new_added_deleted(out var newRows, out var addedRows, out var deletedRows);
+         return new DataPacket<T>(CastRows(newRows), CastRows(addedRows), CastRows(deletedRows));
+
+         IBlueRowCollection<T> CastRows(ICollection rows) =>
+            new BlueTableSelectorWrapper<T>(
+               rows as IBlueRowCollection ?? new BlueRowCollection(rows, dp.Schema), selector);
+      });
+
+   public static IObservable<TableUpdate<IBlueRow, int>> AsTables(this IObservable<IDataPacket> source) =>
+      source.AsTables<IBlueRow, int>(r => r, r => r.Id());
+
+   public static IObservable<TableUpdate<T, int>> AsTables<T>(this IObservable<IDataPacket> source, Func<IBlueRow, T> valueSelector) where T : IBlueRow =>
+      source.AsTables<T, int>(valueSelector, r => r.Id());
+
+   public static IObservable<TableUpdate<T, int>> AsTables<T>(
+      this IObservable<IDataPacket> source, Func<IBlueRow, T> valueSelector, Func<T, int> keySelector) =>
+      source.AsTables<T, int>(valueSelector, keySelector);
+
+   public static IObservable<TableUpdate<T, TKey>> AsTables<T, TKey>(
+      this IObservable<IDataPacket> source, Func<IBlueRow, T> valueSelector, Func<T, TKey> keySelector, Func<IBlueRow, TKey>? rowKeySelector = null) where TKey : IEquatable<TKey> {
+      rowKeySelector ??= r => keySelector(valueSelector(r));
+      return source.Scan(TableUpdate<T, TKey>.mk_empty(keySelector), (res, dp) => {
+         var bd = res.to_builder();
+         dp.get_new_added_deleted(out var newRows, out var addedRows, out var deletedRows);
+         foreach (IBlueRow r in addedRows) {
+            bd.add_or_update(valueSelector(r));
+         }
+         foreach (IBlueRow r in deletedRows) {
+            bd.remove(rowKeySelector(r));
+         }
+         return bd.to_table();
+      });
+   }
+
+   public static IObservable<IDataPacket<T>> AsDataPackets<T>(this IObservable<TableUpdate<T, int>> source, IBlueTableSchema schema) where T : IBlueRow =>
+      source.Scan(
+         (Table: new BlueTableImmutable<T>(schema), Dp: (IDataPacket<T>)DataPacket<T>.Empty(schema)),
+         (res, table) => {
+            var bd = res.Table.ToDataPacketBuilder();
+            foreach (var r in table.removed) {
+               bd.Remove(r.Id());
+            }
+            foreach (var r in table.added) {
+               bd.AddOrUpdate(r);
+            }
+            var resDp = bd.ToDataPacketT(out var newRows, out var addedRows, out var deletedRows);
+            return (newRows, resDp);
+         })
+         .Where(t => t.Dp.addedRows.Any() || t.Dp.deletedRows.Any())
+         .Select(t => t.Dp);
+
+   public static ViewStream AsViewStream<T>(this IObservable<TableUpdate<T, int>> source, IBlueTableSchema schema) where T : IBlueRow =>
+      source.AsDataPackets(schema).AsViewStream();
+
+   public static int ResolveIndices(this IBlueTableSchema schema, string name) {
+      var indices = ResolveIndices(schema, new[] { name });
+      return indices[0];
+   }
+
+   public static (int, int) ResolveIndices(this IBlueTableSchema schema, string name1, string name2) {
+      var indices = ResolveIndices(schema, new[] { name1, name2 });
+      return (indices[0], indices[1]);
+   }
+
+   public static (int, int, int) ResolveIndices(this IBlueTableSchema schema, string name1, string name2, string name3) {
+      var indices = ResolveIndices(schema, new[] { name1, name2, name3 });
+      return (indices[0], indices[1], indices[2]);
+   }
+
+   public static (int, int, int, int) ResolveIndices(this IBlueTableSchema schema, string name1, string name2, string name3, string name4) {
+      var indices = ResolveIndices(schema, new[] { name1, name2, name3, name4 });
+      return (indices[0], indices[1], indices[2], indices[3]);
+   }
+
+   public static int[] ResolveIndices(this IBlueTableSchema schema, string[] columnNames) {
+      var columns = schema.columnInfo;
+      var indices = new int[columnNames.Length];
+      int nResolved = 0;
+      for (int i = 0; i < columnNames.Length; i++) {
+         int origNResolved = nResolved;
+         for (int j = 0; j < columns.Length; j++) {
+            if (columns[j].colName == columnNames[i]) {
+               indices[i] = j;
+               nResolved++;
+               break;
+            }
+         }
+         if (nResolved == origNResolved) {
+            throw new ArgumentException($"No column named '{columnNames[i]}'", nameof(schema));
+         }
+      }
+      return indices;
+   }
 }

src/DataStreamSuite/DataStream/Misc/DSLStream.cs

@@ -3,24 +3,25 @@ namespace bts.SilverPine {
    using BCG;
    using System.Collections.Generic;
    using Bts.DataStreamExpressions;
+   using Bts.DataStream;
 
    // All this does is take in a dsl expression, parse it and fire the data that come out of running it.
    // So for example, if an expresison is passed 
    [DataStreamUserRespresentation("dsl")]
    public class DSLStream : ViewStream {
       private readonly string _dsl;
-      public DSLStream(string dsl) {
+      public DSLStream(string dsl, IDataStreamFactory streamFactory) {
          Log.Gen.info($"DSLStream.cons dsl: {dsl}");
          _dsl = dsl;
-         ViewStream rs = build_stream(dsl);
-         rs = rs ?? new LiteralStream("error", $"msg:invalid dsl \"{dsl}\"");
+         var rs = build_stream(streamFactory, dsl);
+         rs = rs ?? new LiteralStream("error", $"msg:invalid dsl");
          mySetRealSource(rs);
       }
 
-      static ViewStream? build_stream(string stream_expression) {
+      static ViewStream? build_stream(IDataStreamFactory streamFactory, string stream_expression) {
          try {
             var exp = DataStreamExpressionsInterpreter.Evaluate(stream_expression);
-            var rv = exp.getView(new Dictionary<string, StatementExpression>(0));
+            var rv = exp.getView(new Dictionary<string, StatementExpression>(0), streamFactory);
             return rv;
 
          } catch (Exception ex) {
@@ -29,4 +30,4 @@ namespace bts.SilverPine {
          }
       }
    }
-}
+}

src/DataStreamSuite/DataStream/ObservableViewStream.cs

@@ -4,14 +4,21 @@ using BCG;
 
 using bts;
 
+using Microsoft.Extensions.Logging;
+
 using System;
 
 public class ObservableViewStream : ViewStream, IObservable<DataPacket>, IDisposable {
    readonly IObservable<DataPacket> source;
+   readonly string? name;
+   readonly ILogger? logger;
    IDisposable? subscription;
 
-   public ObservableViewStream(IObservable<DataPacket> source) =>
+   public ObservableViewStream(IObservable<DataPacket> source, string? name = null, ILogger? logger = null) {
       this.source = source;
+      this.name = name;
+      this.logger = logger;
+   }
 
    public void Dispose() {
       GC.SuppressFinalize(this);
@@ -23,7 +30,13 @@ public class ObservableViewStream : ViewStream, IObservable<DataPacket>, IDispos
 
    protected override void myOnFirstSubscriber() {
       Log.isnull(subscription);
-      subscription = source.Subscribe(myFireEvent);
+      subscription = source.Subscribe(myFireEvent, OnError, OnCompleted);
+
+      void OnError(Exception exception) =>
+         logger?.LogError("Error in ObservableViewStream. name: {Name}, error: {Exception}", name, exception);
+
+      void OnCompleted() =>
+         logger?.LogInformation("ObservableViewStream completed. name: {Name}", name);
    }
 
    protected override void myOnLastSubscriber() {

src/DataStreamSuite/DataStream/Offloading/RemoteQueryManager.cs

@@ -34,12 +34,12 @@ namespace BCG {
             view_publisher = view_publisher_;
          }
       }
-      static ViewStream? build_stream(string stream_expression)
+      static ViewStream? build_stream(IDataStreamFactory streamFactory, string stream_expression)
       {
          try
          {
             var exp = DataStreamExpressionsInterpreter.Evaluate(stream_expression);
-            var rv = exp.getView(new Dictionary<string, StatementExpression>(0));
+            var rv = exp.getView(new Dictionary<string, StatementExpression>(0), streamFactory);
             return rv;
 
          } catch (Exception ex) {
@@ -51,15 +51,17 @@ namespace BCG {
          _stream_pub_map = new Dictionary<string, StreamPubUnit>();
       readonly IPubTableBroker _broker;
       readonly IManagedSubscriber _sub;
+      readonly IDataStreamFactory streamFactory;
       readonly object syncRoot = new();
       IDisposable? cloudStreamsSub;
       IDisposable? remoteRequestsSub;
       bool isDisposed;
 
 
-      public RemoteQueryManager(IManagedSubscriber managedSubscriber, IPubTableBroker pubTableBroker) {
+      public RemoteQueryManager(IManagedSubscriber managedSubscriber, IPubTableBroker pubTableBroker, IDataStreamFactory streamFactory) {
          _sub = managedSubscriber;
          _broker = pubTableBroker;
+         this.streamFactory = streamFactory;
       }
 
       public Task StartAsync(CancellationToken cancellationToken) {
@@ -102,7 +104,11 @@ namespace BCG {
 
       private void _spawn_remote_stream(string name, string stream_expression, bool persist) {
          bts.Log.Gen.info($"CloudStreamManager.assign name: {name} stream_expression: {stream_expression}");
-         ViewStream stream = build_stream(stream_expression);
+         var stream = build_stream(streamFactory, stream_expression);
+         if (stream is null) {
+            Log.DataFlow.error($"Failed to spawn remote stream. name: {name}; expression: {stream_expression}");
+            return;
+         }
          _expose_to_ws(name, stream);
          if (persist) {
             stream = new PersistorStream(stream, name, "C:\\Temp\\", 1000000);   // 10MBs

src/DataStreamSuite/DataStream/Primitives/SQL/SQLite/SQLiteEnumerator.cs

@@ -0,0 +1,54 @@
+namespace Bts.DataStream;
+
+using BCG;
+
+using System;
+using System.Collections.Generic;
+using System.Data;
+using System.Data.SQLite;
+using System.Linq;
+using System.Threading;
+
+public class SQLiteEnumerable : IAsyncEnumerable<IBlueRowCollection> {
+   readonly string location;
+   readonly string statement;
+   readonly int pageSize;
+
+   public SQLiteEnumerable(string location, string statement, int pageSize = 65536) {
+      this.location = location;
+      this.statement = statement;
+      this.pageSize = pageSize;
+   }
+
+   public async IAsyncEnumerator<IBlueRowCollection> GetAsyncEnumerator(CancellationToken cancellationToken = default) {
+      var dataTable = new DataTable();
+      await using var conn = new SQLiteConnection($"Data Source={location};Version=3");
+      await conn.OpenAsync(cancellationToken);
+      await using var cmd = new SQLiteCommand(statement, conn);
+      await using var reader = cmd.ExecuteReader();
+      var schema = await reader.GetColumnSchemaAsync(cancellationToken);
+      var columns = schema.OrderBy(c => c.ColumnOrdinal ?? int.MaxValue)
+         .Select((c, i) => new ColumnInfo(c.ColumnName, c.ColumnOrdinal ?? i, c.DataType))
+         .Append(new ColumnInfo("id", schema.Count, typeof(int)))
+         .ToArray();
+      var blueSchema = new BlueTableSchema("anon", columns);
+      BlueTable? table = null;
+      int rowId = 1;
+      while (await reader.ReadAsync(cancellationToken)) {
+         table ??= new BlueTable(blueSchema);
+         var rowVals = new object[blueSchema.columnInfo.Length];
+         if (reader.GetValues(rowVals) != rowVals.Length - 1) {
+            throw new InvalidOperationException("Sqlite reader value count for single row not equal to column count");
+         }
+         rowVals[^1] = rowId++;
+         table.addRow(new BlueRow(blueSchema, rowVals));
+         if (table.Count == pageSize) {
+            yield return table;
+            table = null;
+         }
+      }
+      if (table is not null) {
+         yield return table;
+      }
+   }
+}

src/DataStreamSuite/DataStream/TableUpdateDataPacket.cs

@@ -0,0 +1,73 @@
+namespace Bts.DataStream;
+
+using BCG;
+
+using bts.api.core;
+
+using Bts.DataStreamCore;
+
+using System;
+using System.Collections;
+using System.Collections.Generic;
+
+
+public class TableUpdateDataPacket<T> : IDataPacket<T> where T : IBlueRow {
+   readonly TableUpdate<T, int> table;
+   readonly IBlueTableSchema schema;
+
+   public TableUpdateDataPacket(TableUpdate<T, int> table, IBlueTableSchema schema) {
+      this.table = table;
+      this.schema = schema;
+   }
+
+   public IBlueRowCollection<T> newRows {
+      get {
+         var rv = new BlueTableImmutable<T>(schema);
+         return rv;
+      }
+   }
+
+   public IBlueRowCollection<T> addedRows => throw new NotImplementedException();
+
+   public IBlueRowCollection<T> deletedRows => throw new NotImplementedException();
+
+   class MyRowCollection : IBlueRowCollection<T> {
+      readonly TableUpdate<T, int> table;
+      readonly IEnumerable<T> items;
+      Lazy<SortedList<int, T>> itemList;
+
+      public MyRowCollection(IEnumerable<T> items, IBlueTableSchema schema) {
+         this.items = items;
+         this.schema = schema;
+         itemList = new(() => {
+            var rv = new SortedList<int, T>();
+            foreach (var item in this.items) {
+               rv.Add(item.Id(), item);
+            }
+            return rv;
+         });
+      }
+
+      public int Count => itemList.Value.Count;
+      public bool IsSynchronized => false;
+      public object SyncRoot => this;
+
+      public T this[int id] => itemList.Value[id];
+
+      public IBlueTableSchema schema { get; }
+
+      public bool containsID(int id) =>
+         itemList.Value.ContainsKey(id);
+
+      public void CopyTo(Array array, int index) {
+      }
+
+      public IEnumerator<T> GetEnumerator() {
+         throw new NotImplementedException();
+      }
+
+      IEnumerator IEnumerable.GetEnumerator() {
+         throw new NotImplementedException();
+      }
+   }
+}

src/DataStreamSuite/DataStreamCore/BlueData/BlueColumnTable.cs

@@ -0,0 +1,127 @@
+namespace Bts.DataStreamCore;
+
+using BCG;
+
+using bts.utils;
+
+using mbts.utils;
+
+using Spark.Core;
+
+using System;
+
+using System.Collections;
+using System.Collections.Generic;
+using System.Linq;
+
+public class BlueColumnTable : IBlueRowCollection {
+   readonly Variant[][] columns;
+   readonly List<int> rowIds;
+   readonly int idColIndex;
+
+   public BlueColumnTable(IBlueTableSchema schema, int capacity = 4) {
+      ExceptionAssert.Argument.GreaterThanEqualTo(capacity, 0);
+      rowIds = new(capacity);
+      columns = new Variant[schema.columnInfo.Length][];
+      if (columns.Length == 0) {
+         throw new ArgumentException("Schema contains no columns", nameof(schema));
+      }
+      if (!schema.columnInfo.Select(c => c.colName).TryFindIndex("id", out idColIndex)) {
+         throw new ArgumentException("Schema contains no 'id' column", nameof(schema));
+      }
+      this.schema = schema;
+      for (int i = 0; i < columns.Length; i++) {
+         columns[schema.columnInfo[i].colNo] = new Variant[capacity];
+      }
+   }
+
+   public int Count => rowIds.Count;
+   public IBlueTableSchema schema { get; }
+
+   public IBlueRow this[int id] {
+      get {
+         var idx = rowIds.BinarySearch(id);
+         if (idx < 0) {
+            throw new KeyNotFoundException($"No row with id {id}");
+         }
+         return new MyRow(this, idx);
+      }
+   }
+
+   public void Add(Variant[] values) {
+      var id = values[idColIndex].VInt32;
+      var idx = rowIds.BinarySearch(id);
+      if (idx >= 0) {
+         throw new ArgumentException($"Row with id {id} already exists.", nameof(values));
+      }
+      idx = ~idx;
+      for (int i = 0; i < columns.Length; i++) {
+         int len = rowIds.Count;
+         ArrayHelper.Insert(ref columns[i], idx, values[i], ref len);
+      }
+      rowIds.Insert(idx, id);
+   }
+
+   public void AddOrUpdate(Variant[] values) {
+      var id = values[idColIndex].VInt32;
+      var idx = rowIds.BinarySearch(id);
+      if (idx >= 0) {
+         rowIds[idx] = id;
+         for (int i = 0; i < columns.Length; i++) {
+            columns[idx][i] = values[i];
+         }
+         return;
+      }
+      idx = ~idx;
+      int len;
+      for (int i = 0; i < columns.Length; i++) {
+         len = rowIds.Count;
+         ArrayHelper.Insert(ref columns[i], idx, values[i], ref len);
+      }
+      rowIds.Insert(idx, id);
+   }
+
+   public void Clear() =>
+      rowIds.Clear();
+
+   public bool Remove(int id) {
+      var idx = rowIds.BinarySearch(id);
+      if (idx < 0) {
+         return false;
+      }
+      rowIds.RemoveAt(idx);
+      for (int i = 0; i < columns.Length; i++) {
+         ArrayHelper.remove_at(ref columns[i], idx);
+      }
+      return true;
+   }
+
+   public void Trim() {
+      for (int i = 0; i < columns.Length; i++) {
+         ArrayHelper.trim(ref columns[i], rowIds.Count);
+      }
+   }
+
+   public bool containsID(int id) =>
+      rowIds.BinarySearch(id) >= 0;
+
+   public IEnumerator GetEnumerator() {
+      var count = rowIds.Count;
+      for (int i = 0; i < count; i++) {
+         yield return new MyRow(this, i);
+      }
+   }
+
+   class MyRow : IBlueRow {
+      readonly BlueColumnTable table;
+      readonly int rowIdx;
+
+      public MyRow(BlueColumnTable table, int rowIdx) {
+         this.table = table;
+         this.rowIdx = rowIdx;
+      }
+
+      public object this[int colNo] => table.columns[colNo][rowIdx].GetValue();
+      public IBlueTableSchema schema => table.schema;
+   }
+}

src/DataStreamSuite/DataStreamCore/BlueData/BlueDataUtility.cs

@@ -18,6 +18,8 @@ using System.Text.Json.Nodes;
 using Newtonsoft.Json;
 using BCG;
 using System.Runtime.CompilerServices;
+using Spark.Core;
+using System.Numerics;
 
 namespace bts {
 
@@ -89,6 +91,132 @@ namespace BCG {
       public static DateTime get_datetime(this IBlueRow r, int col_no, DateTime err_value = default(DateTime)) {
          return BlueDataUtility.get_row_datetime_value(r, col_no, err_value);
       }
+
+      public static double? GetFloat64(this IBlueRow r, int colNo) =>
+         GetFloat64(r[colNo]);
+
+      public static double? GetFloat64(this IBlueRow r, string colName) =>
+         GetFloat64(r[colName]);
+
+      public static double? GetFloat64(object obj) {
+         if (obj is double d) {
+            return d;
+         }
+         if (obj is int i32) {
+            return i32;
+         }
+         if (obj is long i64) {
+            return i64;
+         }
+         if (obj is string s) {
+            return double.TryParse(s, out d) ? d : null;
+         }
+         if (obj is uint u32) {
+            return u32;
+         }
+         if (obj is ulong u64) {
+            return u64;
+         }
+         if (obj is short i16) {
+            return i16;
+         }
+         if (obj is ushort u16) {
+            return u16;
+         }
+         return null;
+      }
+
+      public static int? GetInt32(this IBlueRow r, int colNo) =>
+         GetInt32(r[colNo]);
+
+      public static int? GetInt32(this IBlueRow r, string colName) =>
+         GetInt32(r[colName]);
+
+      public static int? GetInt32(object obj) {
+         checked {
+            if (obj is double d) {
+               return (int)d;
+            }
+            if (obj is int i32) {
+               return i32;
+            }
+            if (obj is long i64) {
+               return (int)i64;
+            }
+            if (obj is string s) {
+               return double.TryParse(s, out d) ? (int)d : null;
+            }
+            if (obj is uint u32) {
+               return (int)u32;
+            }
+            if (obj is ulong u64) {
+               return (int)u64;
+            }
+            if (obj is short i16) {
+               return i16;
+            }
+            if (obj is ushort u16) {
+               return u16;
+            }
+         }
+         return null;
+      }
+
+      public static long? GetInt64(this IBlueRow r, int colNo) =>
+         GetInt64(r[colNo]);
+
+      public static long? GetInt64(this IBlueRow r, string colName) =>
+         GetInt64(r[colName]);
+
+      public static long? GetInt64(object obj) {
+         checked {
+            if (obj is double d) {
+               return (long)d;
+            }
+            if (obj is int i32) {
+               return i32;
+            }
+            if (obj is long i64) {
+               return i64;
+            }
+            if (obj is string s) {
+               return double.TryParse(s, out d) ? (long)d : null;
+            }
+            if (obj is uint u32) {
+               return u32;
+            }
+            if (obj is ulong u64) {
+               return (long)u64;
+            }
+            if (obj is short i16) {
+               return i16;
+            }
+            if (obj is ushort u16) {
+               return u16;
+            }
+         }
+         return null;
+      }
+
+      public static DateTime? GetUnixEpochDateTime(this IBlueRow row, string colName) =>
+         GetUnixEpochDateTime(row[colName]);
+
+      public static DateTime? GetUnixEpochDateTime(this IBlueRow row, int colNo) =>
+         GetUnixEpochDateTime(row[colNo]);
+
+      public static DateTime? GetUnixEpochDateTime(object obj) {
+         var tsNanos = GetInt64(obj);
+         return tsNanos.HasValue ? DateTime.UnixEpoch.Add(new TimeSpan(tsNanos.Value / 100)) : null;
+      }
+
+      public static DateTime? GetDateTime(object obj) {
+         var res = _get_datetime_value(obj, DateTime.MinValue);
+         if (res == DateTime.MinValue) {
+            return null;
+         }
+         return res;
+      }
+
       public static T get_enum<T>(this IBlueRow r, string col_name, T err_value = default(T)) where T : struct {
          return BlueDataUtility.get_row_enum_value<T>(r, col_name, err_value);
       }
@@ -2224,6 +2352,32 @@ namespace BCG {
          }
       }
 
+      public static TimeSpan? get_timespan(this IBlueRow r, int col_no) =>
+         get_timespan(r[col_no]);
+
+      public static TimeSpan? get_timespan(this IBlueRow r, string col_name) =>
+         get_timespan(r[col_name]);
+
+      public static TimeSpan? get_timespan(object obj) {
+         if (obj is TimeSpan ts) {
+            return ts;
+         }
+         var str = obj as string ?? obj?.ToString();
+         if (str is null) {
+            return null;
+         }
+         if (TimeSpan.TryParse(str, out ts)) {
+            return ts;
+         }
+         if (str.StartsWith("PT")) {
+            try {
+               // Parse ISO 8601 duration
+               return System.Xml.XmlConvert.ToTimeSpan(str);
+            } catch {
+            }
+         }
+         return null;
+      }
       #endregion
 
       #region CSV methods

src/DataStreamSuite/DataStreamCore/BlueData/BlueRowCollection.cs

@@ -0,0 +1,56 @@
+#nullable enable
+namespace Bts.DataStreamCore;
+
+using BCG;
+
+using System;
+using System.Collections;
+using System.Collections.Generic;
+using System.Linq;
+
+public class BlueRowCollection : IBlueRowCollection {
+   readonly ICollection source;
+   readonly Func<int, IBlueRow> findRow;
+   readonly Func<int, bool> containsKey;
+
+   public BlueRowCollection(ICollection source, IBlueTableSchema schema, Func<int, IBlueRow>? findRow = null, Func<int, bool>? containsKey = null) {
+      this.source = source;
+      this.schema = schema;
+      if (findRow is null || containsKey is null) {
+         if (source is IBlueRowCollection brc) {
+            findRow ??= id => brc[id];
+            containsKey ??= brc.containsID;
+         } else if (source is IDictionary dict) {
+            findRow ??= id => dict[id] as IBlueRow ?? throw new KeyNotFoundException();
+            containsKey ??= id => dict.Contains(id);
+         } else if (source is IDictionary<int, IBlueRow> genDict) {
+            findRow ??= id => genDict[id];
+            containsKey ??= genDict.ContainsKey;
+         } else if (source is IReadOnlyDictionary<int, IBlueRow> roGenDict) {
+            findRow ??= id => roGenDict[id];
+            containsKey = roGenDict.ContainsKey;
+         } else {
+            var cachedRows = new Lazy<Dictionary<int, IBlueRow>>(() =>
+               source.Cast<IBlueRow>().ToDictionary(r => r.Id()));
+            findRow ??= id => cachedRows.Value[id];
+            // Don't remove this lambda expression. Intellisense may suggest it, but that would
+            // instantiate 'cachedRows' immediately and defeat the purpose of using Lazy<T>.
+            containsKey ??= id => cachedRows.Value.ContainsKey(id);
+         }
+      }
+      this.findRow = findRow;
+      this.containsKey = containsKey;
+   }
+
+   public IBlueRow this[int id] => findRow(id);
+
+   public IBlueTableSchema schema { get; }
+
+   public int Count => source.Count;
+
+   public bool containsID(int id) =>
+      containsKey(id);
+
+   public IEnumerator GetEnumerator() =>
+      source.GetEnumerator();
+}

src/DataStreamSuite/DataStreamCore/BlueData/BlueRowColumnMask.cs

@@ -0,0 +1,64 @@
+#nullable enable
+namespace Bts.DataStreamCore;
+
+using BCG;
+
+using bts.collections;
+using bts.utils;
+
+using System;
+using System.Collections.Generic;
+
+
+public class BlueRowColumnMask : IBlueRow {
+   readonly IBlueRow row;
+   readonly int[] hiddenColumns;
+
+   public BlueRowColumnMask(IBlueRow row, params int[] hiddenColumns) {
+      this.row = row;
+      this.hiddenColumns = hiddenColumns;
+      Array.Sort(hiddenColumns);
+   }
+
+   /// <summary>
+   /// Provides a way to explicitly skip sorting hiddenColumns. This is done at the callers risk,
+   /// as the index (int) operator performs a binary search on hiddenColumns.
+   /// </summary>
+   /// <param name="row">The underlying row whose columns to hide</param>
+   /// <param name="sortHiddenColumns">Whether or not to sort <paramref name="hiddenColumns"/> in
+   /// the constructor. They are sorted by default to ensure correct behavior in the index (int)
+   /// operator, which performs a binary search on this array.</param>
+   /// <param name="hiddenColumns">The columns (indices) to hide</param>
+   public BlueRowColumnMask(IBlueRow row, bool sortHiddenColumns, params int[] hiddenColumns) {
+      this.row = row;
+      this.hiddenColumns = hiddenColumns;
+      if (sortHiddenColumns) {
+         Array.Sort(hiddenColumns);
+      }
+   }
+
+   public object? this[string colName] {
+      get {
+         for (int i = 0; i < hiddenColumns.Length; i++) {
+            if (row.schema.columnInfo[i].colName == colName) {
+               throw new KeyNotFoundException();
+            }
+         }
+         return row[colName];
+      }
+   }
+   public object? this[int index] {
+      get {
+         if (index > row.schema.columnInfo.Length - hiddenColumns.Length) {
+            throw new IndexOutOfRangeException();
+         }
+         int pos = hiddenColumns.BinarySearch(index);
+         if (pos < 0) {
+            pos = ~pos;
+         }
+         return row[index + pos];
+      }
+   }
+
+   public IBlueTableSchema schema => row.schema;
+}

src/DataStreamSuite/DataStreamCore/BlueData/BlueRowEnumerator.cs

@@ -7,7 +7,7 @@ namespace BCG
    /// Used for enumerating an IEnumerable (ex ICollection). Furthermore, this is the 
    /// default Enueramtor used when enumerating asn IBlueRow
    /// </summary>
-   public class BlueRowEnumerator : IEnumerator
+   public class BlueRowEnumerator : IDictionaryEnumerator
       {
          protected int myPos = -1;
          protected int myRowSize = -1;
@@ -18,6 +18,11 @@ namespace BCG
             myRow = row;
             myRowSize = row.schema.columnNames.Length;
          }
+
+         public DictionaryEntry Entry =>
+            new(myRow.schema.columnInfo[myPos].colName, myRow[myPos]);
+         public object Key => myRow.schema.columnInfo[myPos].colName;
+         public object Value => myRow[myPos];
          
          #region IEnumerator members
          public void Reset()  {myPos = 0;}

src/DataStreamSuite/DataStreamCore/BlueData/BlueRowKeyCollection.cs

@@ -0,0 +1,30 @@
+#nullable enable
+namespace BCG;
+
+using System;
+using System.Collections;
+
+public class BlueRowKeyCollection : ICollection {
+   readonly ColumnInfo[] columns;
+
+   public BlueRowKeyCollection(ColumnInfo[] columns) =>
+      this.columns = columns;
+
+   public int Count => columns.Length;
+
+   public bool IsSynchronized => false;
+
+   public object SyncRoot => this;
+
+   public void CopyTo(Array array, int index) {
+      for (int i = 0; i < columns.Length; i++) {
+         array.SetValue(columns[i].colName, i + index);
+      }
+   }
+
+   public IEnumerator GetEnumerator() {
+      for (int i = 0; i < columns.Length; i++) {
+         yield return columns[i].colName;
+      }
+   }
+}

src/DataStreamSuite/DataStreamCore/BlueData/BlueRowReflection.cs

@@ -0,0 +1,44 @@
+#nullable enable
+namespace Bts.DataStreamCore;
+
+using BCG;
+
+using System.Linq;
+using System.Reflection;
+
+public static class BlueRowReflectionFactory {
+   public static IBlueRow Create<T>(T row, string? tableName = null) =>
+      new BlueRowReflection<T>(row);
+
+   class BlueRowReflection<T> : IBlueRow {
+      public static readonly ColumnInfo[] ColumnInfos;
+      public static readonly PropertyInfo[] ColumnProps;
+
+      readonly T row;
+      readonly string? tableName;
+
+      static BlueRowReflection() {
+         var colTups = typeof(T).GetProperties(BindingFlags.Instance)
+            .Select(p => (Prop: p, Attr: p.GetCustomAttribute<BlueTableColumnAttribute>()))
+            .Where(p => p.Attr is not null)
+            .ToList();
+         ColumnInfos = new ColumnInfo[colTups.Count];
+         ColumnProps = new PropertyInfo[colTups.Count];
+         for (int i = 0; i < colTups.Count; i++) {
+            var (prop, attr) = colTups[i];
+            ColumnInfos[i] = new(attr?.Name ?? prop.Name, i, prop.PropertyType);
+            ColumnProps[i] = prop;
+         }
+      }
+
+      public BlueRowReflection(T row, string? tableName = null) {
+         this.row = row;
+         this.tableName = tableName;
+      }
+
+      public object? this[int index] => ColumnProps[index].GetValue(row);
+
+      public IBlueTableSchema schema =>
+         new BlueTableSchema(ColumnInfos, tableName ?? typeof(T).Name);
+   }
+}

src/DataStreamSuite/DataStreamCore/BlueData/BlueRowStore.cs

@@ -0,0 +1,311 @@
+#nullable enable
+namespace Bts.DataStreamCore;
+
+using BCG;
+
+using mbts.utils;
+
+using System;
+using System.Collections;
+using System.Collections.Generic;
+using System.Diagnostics.CodeAnalysis;
+using System.Linq;
+
+public class BlueRowStore<T1, T2, T3, T4, T5, T6> : IBlueRow, IReadOnlyDictionary<string, object?> {
+   readonly string? name;
+   IBlueTableSchema? _schema;
+
+   public BlueRowStore() { }
+
+   public BlueRowStore(
+      int id,
+      (string Name, T1 Value) item1,
+      (string Name, T2 Value) item2,
+      (string Name, T3 Value) item3,
+      (string Name, T4 Value) item4,
+      (string Name, T5 Value) item5,
+      (string Name, T6 Value) item6
+   ) {
+      Id = id;
+      Item1 = item1;
+      Item2 = item2;
+      Item3 = item3;
+      Item4 = item4;
+      Item5 = item5;
+      Item6 = item6;
+   }
+
+   public int Id { get; set; }
+   public (string Name, T1 Value) Item1 { get; set; }
+   public (string Name, T2 Value) Item2 { get; set; }
+   public (string Name, T3 Value) Item3 { get; set; }
+   public (string Name, T4 Value) Item4 { get; set; }
+   public (string Name, T5 Value) Item5 { get; set; }
+   public (string Name, T6 Value) Item6 { get; set; }
+
+   public int Count => 5;
+   public IReadOnlyList<string> Keys => new KeyList(this);
+   public IReadOnlyList<object?> Values => new ValueList(this);
+
+   public IBlueTableSchema schema =>
+      _schema ??= new BlueTableSchema(name, new ColumnInfo[]{
+         new("id", 0, typeof(int)),
+         new(Item1.Name, 1, typeof(T1)),
+         new(Item2.Name, 2, typeof(T2)),
+         new(Item3.Name, 3, typeof(T3)),
+         new(Item4.Name, 4, typeof(T4)),
+         new(Item5.Name, 5, typeof(T5)),
+         new(Item6.Name, 6, typeof(T6)),
+      });
+
+   ICollection IDictionary.Keys => new KeyList(this);
+   ICollection IDictionary.Values => new ValueList(this);
+
+   IEnumerable<string> IReadOnlyDictionary<string, object?>.Keys => Keys;
+   IEnumerable<object?> IReadOnlyDictionary<string, object?>.Values => Values;
+
+   bool ICollection.IsSynchronized => false;
+   object ICollection.SyncRoot => this;
+
+   public object? this[string colName] {
+      get {
+         if (colName == "id") {
+            return Id;
+         }
+         if (colName == Item1.Name) {
+            return Item1.Value;
+         }
+         if (colName == Item2.Name) {
+            return Item2.Value;
+         }
+         if (colName == Item3.Name) {
+            return Item3.Value;
+         }
+         if (colName == Item4.Name) {
+            return Item4.Value;
+         }
+         if (colName == Item5.Name) {
+            return Item5.Value;
+         }
+         if (colName == Item6.Name) {
+            return Item6.Value;
+         }
+         throw new KeyNotFoundException();
+      }
+      set {
+         if (colName == "id") {
+            Id = ExceptionAssert.Cast(value).To<int>();
+         }
+         if (colName == Item1.Name) {
+            Item1 = (Item1.Name, ExceptionAssert.Cast(value).To<T1>());
+         }
+         if (colName == Item2.Name) {
+            Item2 = (Item2.Name, ExceptionAssert.Cast(value).To<T2>());
+         }
+         if (colName == Item3.Name) {
+            Item3 = (Item3.Name, ExceptionAssert.Cast(value).To<T3>());
+         }
+         if (colName == Item4.Name) {
+            Item4 = (Item4.Name, ExceptionAssert.Cast(value).To<T4>());
+         }
+         if (colName == Item5.Name) {
+            Item5 = (Item5.Name, ExceptionAssert.Cast(value).To<T5>());
+         }
+         if (colName == Item6.Name) {
+            Item6 = (Item6.Name, ExceptionAssert.Cast(value).To<T6>());
+         }
+         throw new KeyNotFoundException();
+      }
+   }
+
+   public object? this[int index] => index switch {
+      0 => Id,
+      1 => Item1.Value,
+      2 => Item2.Value,
+      3 => Item3.Value,
+      4 => Item4.Value,
+      5 => Item5.Value,
+      6 => Item6.Value,
+      _ => throw new IndexOutOfRangeException()
+   };
+
+   object? IDictionary.this[object key] {
+      get => this[ExceptionAssert.Cast(key).To<string>()];
+      set => this[ExceptionAssert.Cast(key).To<string>()] = value;
+   }
+
+   public void CopyTo(Array array, int index) {
+      array.SetValue(Id, index++);
+      array.SetValue(Item1.Value, index++);
+      array.SetValue(Item2.Value, index++);
+      array.SetValue(Item3.Value, index++);
+      array.SetValue(Item4.Value, index++);
+      array.SetValue(Item5.Value, index++);
+      array.SetValue(Item6.Value, index++);
+   }
+
+   public bool Contains(object key) =>
+      key is string keystr && ContainsKey(keystr);
+
+   public bool ContainsKey(string key) =>
+      Keys.Contains(key);
+
+   public void SetId(int id) =>
+      Id = id;
+
+   public void SetValue1(T1 value) =>
+      Item1 = (Item1.Name, value);
+
+   public void SetValue1(string colName, T1 value) =>
+      Item1 = (colName, value);
+
+   public void SetValue2(T2 value) =>
+      Item2 = (Item2.Name, value);
+
+   public void SetValue2(string colName, T2 value) =>
+      Item2 = (colName, value);
+
+   public void SetValue3(T3 value) =>
+      Item3 = (Item3.Name, value);
+
+   public void SetValue3(string colName, T3 value) =>
+      Item3 = (colName, value);
+
+   public void SetValue4(T4 value) =>
+      Item4 = (Item4.Name, value);
+
+   public void SetValue4(string colName, T4 value) =>
+      Item4 = (colName, value);
+
+   public void SetValue5(T5 value) =>
+      Item5 = (Item5.Name, value);
+
+   public void SetValue5(string colName, T5 value) =>
+      Item5 = (colName, value);
+
+   public void SetValue6(T6 value) =>
+      Item6 = (Item6.Name, value);
+
+   public void SetValue6(string colName, T6 value) =>
+      Item6 = (colName, value);
+
+   public bool TryGetValue(string key, [MaybeNullWhen(false)] out object? value) {
+      if (!Keys.TryFindIndex(key, out int index)) {
+         value = null;
+         return false;
+      }
+      value = this[index];
+      return true;
+   }
+
+   void IDictionary.Add(object key, object? value) =>
+      throw new NotSupportedException();
+
+   void IDictionary.Clear() =>
+      throw new NotSupportedException();
+
+   void IDictionary.Remove(object key) =>
+      throw new NotSupportedException();
+
+   IDictionaryEnumerator IDictionary.GetEnumerator() =>
+      throw new NotImplementedException();
+
+   IEnumerator IEnumerable.GetEnumerator() =>
+      throw new NotImplementedException();
+
+   public IEnumerator<KeyValuePair<string, object?>> GetEnumerator() {
+      throw new NotImplementedException();
+   }
+
+   class KeyList : ICollection, IReadOnlyList<string> {
+      readonly BlueRowStore<T1, T2, T3, T4, T5, T6> store;
+
+      public KeyList(BlueRowStore<T1, T2, T3, T4, T5, T6> store) =>
+         this.store = store;
+
+      public int Count => 5;
+      bool ICollection.IsSynchronized => false;
+      object ICollection.SyncRoot => this;
+
+      public string this[int index] =>
+         index switch {
+            0 => "id",
+            1 => store.Item1.Name,
+            2 => store.Item2.Name,
+            3 => store.Item3.Name,
+            4 => store.Item4.Name,
+            5 => store.Item5.Name,
+            6 => store.Item6.Name,
+            _ => throw new IndexOutOfRangeException()
+         };
+
+      public void CopyTo(Array array, int index) {
+         array.SetValue(store.Id, index++);
+         array.SetValue(store.Item1.Value, index++);
+         array.SetValue(store.Item2.Value, index++);
+         array.SetValue(store.Item3.Value, index++);
+         array.SetValue(store.Item4.Value, index++);
+         array.SetValue(store.Item5.Value, index++);
+         array.SetValue(store.Item6.Value, index++);
+      }
+
+      public IEnumerator<string> GetEnumerator() {
+         yield return "id";
+         yield return store.Item1.Name;
+         yield return store.Item2.Name;
+         yield return store.Item3.Name;
+         yield return store.Item4.Name;
+         yield return store.Item5.Name;
+         yield return store.Item6.Name;
+      }
+
+      IEnumerator IEnumerable.GetEnumerator() =>
+         GetEnumerator();
+   }
+
+   class ValueList : ICollection, IReadOnlyList<object?> {
+      readonly BlueRowStore<T1, T2, T3, T4, T5, T6> store;
+
+      public ValueList(BlueRowStore<T1, T2, T3, T4, T5, T6> store) =>
+         this.store = store;
+
+      public object? this[int index] =>
+         index switch {
+            0 => store.Id,
+            1 => store.Item1.Value,
+            2 => store.Item2.Value,
+            3 => store.Item3.Value,
+            4 => store.Item4.Value,
+            5 => store.Item5.Value,
+            6 => store.Item6.Value,
+            _ => throw new IndexOutOfRangeException()
+         };
+
+      public int Count => 5;
+      public bool IsSynchronized => false;
+      public object SyncRoot => this;
+
+      public void CopyTo(Array array, int index) {
+         array.SetValue(store.Id, index++);
+         array.SetValue(store.Item1.Value, index++);
+         array.SetValue(store.Item2.Value, index++);
+         array.SetValue(store.Item3.Value, index++);
+         array.SetValue(store.Item4.Value, index++);
+         array.SetValue(store.Item5.Value, index++);
+         array.SetValue(store.Item6.Value, index++);
+      }
+
+      public IEnumerator<object?> GetEnumerator() {
+         yield return store.Id;
+         yield return store.Item1.Value;
+         yield return store.Item2.Value;
+         yield return store.Item3.Value;
+         yield return store.Item4.Value;
+         yield return store.Item5.Value;
+         yield return store.Item6.Value;
+      }
+
+      IEnumerator IEnumerable.GetEnumerator() =>
+         GetEnumerator();
+   }
+}

src/DataStreamSuite/DataStreamCore/BlueData/BlueRowValueCollection.cs

@@ -0,0 +1,32 @@
+#nullable enable
+namespace BCG;
+
+using System;
+using System.Collections;
+
+public class BlueRowValueCollection : ICollection {
+   readonly IBlueRow row;
+
+   public BlueRowValueCollection(IBlueRow row) =>
+      this.row = row;
+
+   public int Count => row.schema.columnInfo.Length;
+
+   public bool IsSynchronized => false;
+
+   public object SyncRoot => this;
+
+   public void CopyTo(Array array, int index) {
+      var length = row.schema.columnInfo.Length;
+      for (int i = 0; i < length; i++) {
+         array.SetValue(row[i], i + index);
+      }
+   }
+
+   public IEnumerator GetEnumerator() {
+      var length = row.schema.columnInfo.Length;
+      for (int i = 0; i < length; i++) {
+         yield return row[i];
+      }
+   }
+}

src/DataStreamSuite/DataStreamCore/BlueData/BlueTableColumnAttribute.cs

@@ -0,0 +1,17 @@
+#nullable enable
+namespace Bts.DataStreamCore;
+
+using System;
+
+[AttributeUsage(AttributeTargets.Property)]
+public class BlueTableColumnAttribute : Attribute {
+   readonly int index;
+
+   public BlueTableColumnAttribute(string? name = null, int index = -1) {
+      Name = name;
+      this.index = index;
+   }
+
+   public string? Name { get; }
+   public int? Index => index >= 0 ? index : null;
+}

src/DataStreamSuite/DataStreamCore/BlueData/BlueTableImmutable.cs

@@ -196,6 +196,24 @@ public class BlueTableImmutable<T> : IBlueRowCollection<T> where T : IBlueRow {
          return new DataPacket<T>(newRows, addedRows, removedRows);
       }
 
+      public IDataPacket<T> ToDataPacketT(
+         out BlueTableImmutable<T> newRows,
+         out BlueTableImmutable<T> addedRows,
+         out BlueTableImmutable<T> removedRows,
+         bool reset = true
+      ) {
+         table.RemoveRange(removed.Keys);
+         table.AddRange(added);
+         newRows = new BlueTableImmutable<T>(schema, table.ToImmutable());
+         addedRows = new BlueTableImmutable<T>(schema, added.ToImmutable());
+         removedRows = new BlueTableImmutable<T>(schema, removed.ToImmutable());
+         if (reset) {
+            added.Clear();
+            removed.Clear();
+         }
+         return new DataPacket<T>(newRows, addedRows, removedRows);
+      }
+
       public BlueTableImmutable<T> ToImmutable() =>
          new(schema, table.ToImmutable());
    }

src/DataStreamSuite/DataStreamCore/BlueData/BlueTableNameAttribute.cs

@@ -0,0 +1,12 @@
+#nullable enable
+namespace Bts.DataStreamCore;
+
+using System;
+
+[AttributeUsage(AttributeTargets.Class)]
+public class BlueTableNameAttribute : Attribute {
+   public BlueTableNameAttribute(string name) =>
+      Name = name;
+
+   public string Name { get; }
+}

src/DataStreamSuite/DataStreamCore/BlueData/BlueTableSchemaBuilder.cs

@@ -0,0 +1,40 @@
+#nullable enable
+namespace Bts.DataStreamCore;
+
+using BCG;
+
+using System;
+using System.Collections;
+using System.Collections.Generic;
+
+public class BlueTableSchemaBuilder : IEnumerable<(string, Type)> {
+   readonly string? tableName;
+   readonly List<(string Name, Type Type)> colDefs = new();
+
+   public BlueTableSchemaBuilder(string? tableName = null) =>
+      this.tableName = tableName;
+
+   public void Add((string Name, Type Type) tup) =>
+      Add(tup.Name, tup.Type);
+
+   public void Add<T>(string Name) =>
+      colDefs.Add((Name, typeof(T)));
+
+   public void Add(string Name, Type Type) =>
+      colDefs.Add((Name, Type));
+
+   public IEnumerator<(string, Type)> GetEnumerator() {
+      throw new NotImplementedException();
+   }
+
+   public IBlueTableSchema ToSchema(string? tableName = null) {
+      var colInfos = new ColumnInfo[colDefs.Count];
+      for (int i = 0; i < colInfos.Length; i++) {
+         colInfos[i] = new(colDefs[i].Name, i, colDefs[i].Type);
+      }
+      return new BlueTableSchema(colInfos, tableName ?? this.tableName);
+   }
+
+   IEnumerator IEnumerable.GetEnumerator() =>
+      colDefs.GetEnumerator();
+}

src/DataStreamSuite/DataStreamCore/BlueData/BlueTableSchemaFactory.cs

@@ -0,0 +1,16 @@
+#nullable enable
+namespace Bts.DataStreamCore;
+
+using BCG;
+
+using System;
+
+public class BlueTableSchemaFactory {
+   public static IBlueTableSchema Create(string tableName, params (string Name, Type Type)[] columns) {
+      var colInfos = new ColumnInfo[columns.Length];
+      for (int i = 0; i < columns.Length; i++) {
+         colInfos[i] = new(columns[i].Name, i, columns[i].Type);
+      }
+      return new BlueTableSchema(colInfos, tableName);
+   }
+}

src/DataStreamSuite/DataStreamCore/BlueData/BlueTableSelectorWrapper.cs

@@ -0,0 +1,39 @@
+#nullable enable
+namespace Bts.DataStreamCore;
+
+using BCG;
+
+using System;
+using System.Collections.Generic;
+
+public static class BlueTableSelectorWrapper {
+   public static BlueTableSelectorWrapper<T> Create<T>(
+      IBlueRowCollection source, Func<IBlueRow, T> selector, IBlueTableSchema? schema = null) where T : IBlueRow =>
+      new(source, selector, schema);
+}
+
+public class BlueTableSelectorWrapper<T> : IBlueRowCollection<T> where T : IBlueRow {
+   readonly IBlueRowCollection source;
+   readonly Func<IBlueRow, T> selector;
+
+   public BlueTableSelectorWrapper(IBlueRowCollection source, Func<IBlueRow, T> selector, IBlueTableSchema? schema = null) {
+      this.source = source;
+      this.selector = selector;
+      this.schema = schema ?? source.schema;
+   }
+
+   public T this[int id] => selector(source[id]);
+
+   public IBlueTableSchema schema { get; }
+
+   public int Count => source.Count;
+
+   public bool containsID(int id) =>
+      source.containsID(id);
+
+   public IEnumerator<T> GetEnumerator() {
+      foreach (IBlueRow r in source) {
+         yield return selector(r);
+      }
+   }
+}

src/DataStreamSuite/DataStreamCore/BlueData/ColumnIndex.cs

@@ -0,0 +1,64 @@
+namespace Bts.DataStreamCore;
+
+using BCG;
+
+using System;
+
+public struct ColumnIndex {
+   public readonly string[] ColumnNames;
+   public readonly int[] ColumnIndices;
+
+   public ColumnIndex(params string[] columnNames) {
+      ColumnNames = columnNames;
+      ColumnIndices = new int[columnNames.Length];
+   }
+
+   public readonly int this[int nameIndex] => ColumnIndices[nameIndex];
+
+   public static int Resolve(ColumnInfo[] columns, string name) {
+      var indices = Resolve(columns, new[] { name });
+      return indices[0];
+   }
+
+   public static (int, int) Resolve(ColumnInfo[] columns, string name1, string name2) {
+      var indices = Resolve(columns, new[] { name1, name2 });
+      return (indices[0], indices[1]);
+   }
+
+   public static int[] Resolve(ColumnInfo[] columns, params string[] names) {
+      var indices = new int[names.Length];
+      int nResolved = 0;
+      for (int i = 0; i < names.Length; i++) {
+         int origNResolved = nResolved;
+         for (int j = 0; j < columns.Length; j++) {
+            if (columns[j].colName == names[i]) {
+               indices[i] = j;
+               nResolved++;
+               break;
+            }
+         }
+         if (nResolved == origNResolved) {
+            throw new ArgumentException($"No column named '{names[i]}'", nameof(columns));
+         }
+      }
+      return indices;
+   }
+
+   public int Resolve(ColumnInfo[] columns, bool requireSuccess = true) {
+      int nResolved = 0;
+      for (int i = 0; i < ColumnNames.Length; i++) {
+         int origNResolved = nResolved;
+         for (int j = 0; j < columns.Length; j++) {
+            if (columns[j].colName == ColumnNames[i]) {
+               ColumnIndices[i] = j;
+               nResolved++;
+               break;
+            }
+         }
+         if (requireSuccess && nResolved == origNResolved) {
+            throw new ArgumentException($"No column named '{ColumnNames[i]}'", nameof(columns));
+         }
+      }
+      return nResolved;
+   }
+}

src/DataStreamSuite/DataStreamCore/BlueData/IBlueRow.cs

@@ -1,6 +1,8 @@
 #nullable enable
 using System;
 using System.Collections;
+using System.Collections.Generic;
+using System.Linq;
 
 namespace BCG
 {
@@ -23,11 +25,50 @@ namespace BCG
       /// half a million type checkings
       /// </summary>
       object? this[int index] {get;}
-      object? this[string colName] {get;}
+      object? this[string colName] {
+         get {
+            var cols = schema.columnInfo;
+            for (int i = 0; i < cols.Length; i++) {
+               if (cols[i].colName == colName) {
+                  return this[i];
+               }
+            }
+            throw new KeyNotFoundException();
+         }
+      }
+
+      object? IDictionary.this[object key] {
+         get => key is string s
+            ? this[s]
+            : throw new ArgumentException($"Key must be of type string (actually {key?.GetType()})", nameof(key));
+         set => throw new NotImplementedException();
+      }
+
+      ICollection IDictionary.Keys => new BlueRowKeyCollection(schema.columnInfo);
+      ICollection IDictionary.Values => new BlueRowValueCollection(this);
+
+      int ICollection.Count => schema.columnInfo.Length;
       bool IDictionary.IsFixedSize => true;
       bool IDictionary.IsReadOnly => true;
       object ICollection.SyncRoot => this;
       bool ICollection.IsSynchronized => false;
+      bool IDictionary.Contains(object key) {
+         if (key is string s) {
+            return schema.columnNames.Contains(s);
+         }
+         return false;
+      }
+      void ICollection.CopyTo(Array array, int index) {
+         var length = schema.columnInfo.Length;
+         for (int i = 0; i < length; i++) {
+            array.SetValue(this[i], i + index);
+         }
+      }
+      void IDictionary.Add(object key, object? value) => throw new NotImplementedException();
+      void IDictionary.Clear() => throw new NotImplementedException();
+      IDictionaryEnumerator IDictionary.GetEnumerator() => new BlueRowEnumerator(this);
+      IEnumerator IEnumerable.GetEnumerator() => ((IDictionary)this).GetEnumerator();
+      void IDictionary.Remove(object key) => throw new NotSupportedException();
    }
 
    // public interface IBlueRow<T> : IBlueRow where T : IBlueRow {

src/DataStreamSuite/DataStreamCore/BlueData/IBlueRowCollection.cs

@@ -16,11 +16,23 @@ namespace BCG
       IBlueRow this[int id]{get;}
       IBlueTableSchema schema {get;}
       bool containsID (int id);
+
+      bool ICollection.IsSynchronized => false;
+      object ICollection.SyncRoot => this;
+
+      void ICollection.CopyTo(Array array, int index) {
+         foreach (IBlueRow r in this) {
+            array.SetValue(r, index++);
+         }
+      }
    }
 
    public interface IBlueRowCollection<T> : IBlueRowCollection, IReadOnlyCollection<T> where T : IBlueRow {
       new T this[int id] { get; }
 
       IBlueRow IBlueRowCollection.this[int id] => this[id];
+
+      IEnumerator IEnumerable.GetEnumerator() =>
+         GetEnumerator();
    }
 }

src/DataStreamSuite/DataStreamCore/DataPacket.cs

@@ -3,6 +3,8 @@ using System.Collections;
 using System.Data;
 using BCG.Utilities.General;
 
+using Bts.DataStreamCore;
+
 namespace BCG
 {
 	[Serializable]
@@ -183,6 +185,7 @@ namespace BCG
             return retVal;
          }
       }
+      IBlueTableSchema IDataPacket.Schema => schema;
 
       /*          
        status  D total  A total  D this   A This   D' total A' Total 
@@ -324,6 +327,12 @@ namespace BCG
       public IBlueRowCollection<T> newRows { get; }
       public IBlueRowCollection<T> addedRows { get; }
       public IBlueRowCollection<T> deletedRows { get; }
+
+      public static DataPacket<T> Empty(IBlueTableSchema schema) =>
+         new(
+            new BlueTableImmutable<T>(schema),
+            new BlueTableImmutable<T>(schema),
+            new BlueTableImmutable<T>(schema));
    }
 
    public static class DataPacketExtensions {

src/DataStreamSuite/DataStreamCore/DataPacketJsonConverter.cs

@@ -129,6 +129,12 @@ public class DataPacketJsonConverter : JsonConverter<DataPacket> {
          if (t == typeof(bool)) {
             return NullCheckWrapper(v => writer.WriteBooleanValue((bool)v));
          }
+         if (t == typeof(DateTime)) {
+            return NullCheckWrapper(v => writer.WriteStringValue(((DateTime)v).ToUniversalTime().ToString("yyyy-MM-ddTHH:mm:ss.ffffff")));
+         }
+         if (t == typeof(TimeSpan)) {
+            return NullCheckWrapper(v => writer.WriteStringValue(((TimeSpan)v).ToString(@"hh\:mm\:ss\.ffffff")));
+         }
          return NullCheckWrapper(v => writer.WriteStringValue(v?.ToString()));
 
          Action<object?> NullCheckWrapper(Action<object> writeNonNull) =>

src/DataStreamSuite/DataStreamCore/DynamicLoading/CompositeDataStreamFactory.cs

@@ -0,0 +1,29 @@
+#nullable enable
+namespace Bts.DataStream;
+
+using BCG;
+
+using bts.SilverPine;
+
+using System.Collections.Generic;
+using System.Linq;
+
+public class CompositeDataStreamFactory : IDataStreamFactory {
+   readonly List<IDataStreamFactory> streamFactories;
+
+   public CompositeDataStreamFactory(IEnumerable<IDataStreamFactory> streamFactories) {
+      this.streamFactories = streamFactories.ToList();
+      foreach (var rsf in this.streamFactories.OfType<IRecursiveDataStreamFactory>()) {
+         rsf.OnCompositeFactory(this);
+      }
+   }
+
+   public IEnumerable<IDataStreamMetadata> GetSupportedStreams() =>
+      streamFactories.SelectMany(f => f.GetSupportedStreams());
+
+   public ViewStream? Create(string streamId, IEnumerable<(string? Name, int? Index, object? Value)> args) =>
+      streamFactories.Where(f => f.CanCreate(streamId))
+         .Select(f => f.Create(streamId, args))
+         .ExceptNull()
+         .FirstOrDefault();
+}

src/DataStreamSuite/DataStreamCore/DynamicLoading/DataStreamMetadata.cs

@@ -0,0 +1,52 @@
+#nullable enable
+namespace bts.SilverPine;
+
+using BCG;
+
+using System;
+using System.Collections.Generic;
+using System.Linq;
+using System.Reflection;
+
+public interface IDataStreamMetadata {
+   public string Name { get; }
+   public IEnumerable<string> AlternateNames { get; }
+}
+
+public class DataStreamMetadata : IDataStreamMetadata {
+   public string? Name { get; set; }
+   public List<string>? AlternateNames { get; set; }
+
+   string IDataStreamMetadata.Name => Name ?? string.Empty;
+   IEnumerable<string> IDataStreamMetadata.AlternateNames =>
+      AlternateNames ?? new(0);
+
+   public static DataStreamMetadata FromType(Type type) {
+      var friendlyName = type.GetCustomAttribute<DataStreamUserRespresentation>()?.friendlyName;
+      var rv = new DataStreamMetadata() {
+         Name = friendlyName ?? type.Name,
+         AlternateNames = new(2) {
+            Log.notnull(type.FullName)
+         }
+      };
+      if (friendlyName is not null) {
+         rv.AlternateNames.Add(type.Name);
+      }
+      return rv;
+   }
+}
+
+public static class DataStreamMetadataExtensions {
+   public static bool IsMatch(this IDataStreamMetadata dsm, string name, bool checkAlternateNames = true, bool ignoreCase = true, bool ignoreUnderscores = true) {
+      var strComp = ignoreCase ? StringComparison.CurrentCultureIgnoreCase : StringComparison.CurrentCulture;
+      var nameWithoutUnderscores = new Lazy<string>(() => RemoveUnderscores(name));
+      return IsNameMatch(dsm.Name) || (checkAlternateNames && dsm.AlternateNames.Any(IsNameMatch));
+
+      static string RemoveUnderscores(string s) =>
+         s.Replace("_", "");
+
+      bool IsNameMatch(string otherName) =>
+         string.Equals(name, otherName, strComp)
+            || (ignoreUnderscores && string.Equals(nameWithoutUnderscores.Value, RemoveUnderscores(otherName), strComp));
+   }
+}

src/DataStreamSuite/DataStreamCore/DynamicLoading/IDataStreamFactory.cs

@@ -0,0 +1,27 @@
+#nullable enable
+namespace Bts.DataStream;
+
+using BCG;
+
+using bts.SilverPine;
+
+using System.Collections.Generic;
+using System.Linq;
+
+public interface IDataStreamFactory {
+   IEnumerable<IDataStreamMetadata> GetSupportedStreams();
+
+   ViewStream? Create(string streamId, IEnumerable<(string? Name, int? Index, object? Value)> args);
+
+   bool CanCreate(string streamId) =>
+      GetSupportedStreams().Any(s => s.IsMatch(streamId));
+}
+
+/// <summary>
+/// This interface provides a way for IDataStreamFactory's that create ViewStreams
+/// that depend specifically on the composite IDataStreamFactory. This was designed
+/// specifically for the NLStreamDataFactory.
+/// </summary>
+public interface IRecursiveDataStreamFactory : IDataStreamFactory {
+   void OnCompositeFactory(IDataStreamFactory compositeFactory);
+}

src/DataStreamSuite/DataStreamCore/DynamicLoading/PrimitiveManager.cs

@@ -9,31 +9,36 @@ namespace BCG {
    using System.Reflection;
    using System.IO;
    using System.Collections.Generic;
+   using System.Linq;
+   using bts.collections;
+   using System.Diagnostics.CodeAnalysis;
+   using bts.SilverPine;
+   using Spark.Core.Collections;
+   using Bts.DataStreamCore;
+   using Microsoft.Extensions.Options;
 
-   public class PrimitiveManager {
+   public class PrimitiveManager : IDataStreamFactory {
       private static PrimitiveManager mySelf;
       readonly List<string> myAssemblyNames = new();
       readonly Dictionary<string, Type> myTypeLookupTable = new();
 
-      private PrimitiveManager() {
-         myLoadAllTypes();
-      }
+      public PrimitiveManager(IOptions<PrimitiveManagerConfig> config) : this(config.Value) { }
 
-      private void myLoadAllTypes() {
+      public PrimitiveManager(PrimitiveManagerConfig config) {
          AppDomain currentDomain = AppDomain.CurrentDomain;
          currentDomain.AssemblyResolve += new ResolveEventHandler(myAssemblyResolver);
          loadAssembly(this.GetType().Assembly.FullName);
-         var assemblies = new string[] { "DataStream.dll", "SharedStreams.dll" };
-         var config_assemblies = Configuration.get<DataStreamConfig>().Assemblies;
-         bts.Log.Gen.info($"PrimitiveManager.myLoadAllTypes config_assemblies: {config_assemblies}");
-         foreach (string assemblyName in assemblies/*config_assemblies*/) {
+         foreach (string assemblyName in config.Assemblies.empty_if_null()) {
             bts.Log.Gen.info($"PrimitiveManager.myLoadAllTypes assemblyName: {assemblyName}");
             string resolvedAssemblyPath = assemblyName;
             if (!System.IO.Path.IsPathRooted(assemblyName)) {
                resolvedAssemblyPath = Path.GetFullPath(Path.Combine(AppDomain.CurrentDomain.BaseDirectory, assemblyName));
             }
-            loadAssembly(resolvedAssemblyPath);
+            loadAssembly(resolvedAssemblyPath, IncludeType);
          }
+
+         bool IncludeType(Type t) =>
+            !config.ExcludeTypes.empty_if_null().Any(tname => tname == t.Name);
       }
 
       private static Assembly myAssemblyResolver(object sender, ResolveEventArgs args) {
@@ -47,7 +52,7 @@ namespace BCG {
       /// Loads all the Stream types associated with the assembly
       /// </summary>
       /// <param name="assemblyName"></param>
-      public void loadAssembly(string assemblyName) {
+      public void loadAssembly(string assemblyName, Func<Type, bool> typeFilter = null) {
          bts.Log.Gen.info($"PrimitiveManager.loadAssembly assemblyName: {assemblyName}");
          if (myAssemblyNames.Contains(assemblyName))
             return;
@@ -56,7 +61,11 @@ namespace BCG {
             GetType().Assembly :
             Assembly.LoadFrom(assemblyName);
 
-         foreach (Type type in loadedAssembly.GetTypes())
+         IEnumerable<Type> types = loadedAssembly.GetTypes();
+         if (typeFilter is not null) {
+            types = types.Where(typeFilter);
+         }
+         foreach (Type type in types)
             if (!type.IsInterface
                && !type.IsAbstract
                && type.IsSubclassOf(typeof(ViewStream))
@@ -73,7 +82,7 @@ namespace BCG {
                      break;
                   }
                if (isVisible) {
-                  Log.assert_d(myTypeLookupTable[userRep] == null, type.FullName);
+                  Log.assert_d(!myTypeLookupTable.ContainsKey(userRep));
                   myTypeLookupTable[userRep] = type;
                }
             }
@@ -89,22 +98,30 @@ namespace BCG {
          return retVal;
       }
 
+      public IEnumerable<Type> GetSupportedTypes() =>
+         myTypeLookupTable.Values.Cast<Type>();
 
+      public IEnumerable<IDataStreamMetadata> GetSupportedStreams() =>
+         GenericDictionaryWrapper.Create<string, Type>(myTypeLookupTable).Select(kv => new DataStreamMetadata() {
+            Name = kv.Key,
+            AlternateNames = new() { kv.Value.Name, kv.Value.FullName }
+         });
 
-      public static PrimitiveManager instance {
-         get { return mySelf = mySelf == null ? new PrimitiveManager() : mySelf; }
+      public Type ResolveType(string streamId) =>
+         TryResolveType(streamId, out var t) ? t
+            : throw new ArgumentException($"Failed to resolve ViewStream type '{streamId}'", nameof(streamId));
+
+      public bool TryResolveType(string streamId, [NotNullWhen(true)] out Type result) {
+         result = GetSupportedStreams()
+            .Where(s => s.IsMatch(streamId))
+            .Select(s => s.Name)
+            .Prepend(streamId)
+            .Select(myTypeLookupTable.GetValueOrDefault)
+            .ExceptNull()
+            .FirstOrDefault();
+         return result is not null;
       }
 
-      /// <summary>
-      /// Returns the Type corresponding to the primitive name
-      /// </summary>
-      /// <param name="friendlyName"></param>
-      /// <returns></returns>
-      public Type find(string friendlyName) =>
-         myTypeLookupTable.TryGetValue(friendlyName, out var t)
-            ? t
-            : throw new ArgumentException($"'{friendlyName}' is not the friendly name of any of the loaded types", nameof(friendlyName));
-
       /// <summary>
       /// Constructs and returns a ViewStream of type corresponding to the friendly name
       /// </summary>
@@ -118,8 +135,110 @@ namespace BCG {
             constructorArgs = new object[args.Count];
             args.CopyTo(constructorArgs, 0);
          }
-         var type = find(friendlyName);
-         return Activator.CreateInstance(type, constructorArgs) as ViewStream;
+         if (myTypeLookupTable[friendlyName] == null)
+            throw new ApplicationException(friendlyName + " is not the friendly name of any of the loaded types");
+         try {
+            return
+               constructorArgs == null ?
+               Activator.CreateInstance(myTypeLookupTable[friendlyName] as Type) as ViewStream :
+               Activator.CreateInstance(myTypeLookupTable[friendlyName] as Type, constructorArgs) as ViewStream;
+         } catch (Exception ex) {
+            Log.Gen.warn($"PrimitiveManager failed to construct ViewStream type '{friendlyName}': {ex}");
+         }
+         return null;
+      }
+
+      public ViewStream Create(string streamId, IEnumerable<(string Name, int? Index, object Value)> args) =>
+         TryResolveType(streamId, out var type) ? Create(type, args) : null;
+
+      /// <summary>
+      /// Constructs and returns a ViewStream of type corresponding to the friendly name
+      /// </summary>
+      /// <param name="friendlyName">friendly primitive name </param>
+      /// <param name="args">args can be null or an empty ICollection if the paramterless constructor
+      /// is the desired one</param>
+      /// <returns></returns>
+      public ViewStream Create(Type type, IEnumerable<(string Name, int? Index, object Value)> args) {
+         var ctrs = type.GetConstructors(BindingFlags.Instance | BindingFlags.Public);
+         var argList = args.ToList();
+         const int MAX_ALLOWED_ARGS = 64;
+         if (argList.Count > MAX_ALLOWED_ARGS) {
+            throw new ArgumentException(
+               $"ViewStream arg count ({argList.Count}) exceeds max allowed ({MAX_ALLOWED_ARGS})", nameof(args));
+         }
+         if (argList.Any(a => !(a.Name?.Length > 0) && !a.Index.HasValue)) {
+            throw new ArgumentException(
+               "One or more ViewStream args lack both Name and Index (unable to assign a constructor param)", nameof(args));
+         }
+         var indexedArgs = argList.Where(a => a.Index.HasValue).ToList();
+         var namedArgs = argList.Where(a => a.Name?.Length > 0).ToList();
+         if (!namedArgs.Select(a => a.Name).IsDistinct()) {
+            throw new ArgumentException("ViewStream arg names are not unique");
+         }
+         if (!indexedArgs.Select(a => a.Index.Value).IsDistinct()) {
+            throw new ArgumentException("ViewStream arg indices are not unique");
+         }
+         // Constructor params with default values may be omitted from the input 'args'. Therefore,
+         // 'minArgCount' >= 'argList.Count'.
+         var minArgCount = Math.Max(indexedArgs.Select(a => a.Index.Value).MaxOrDefault(), argList.Count);
+         foreach (var ctr in ctrs) {
+            if (TryAssignConstructorArgs(ctr, minArgCount, argList, indexedArgs, namedArgs, out var ctrArgs)) {
+               return ctr.Invoke(ctrArgs) as ViewStream;
+            }
+         }
+         return null;
+
+         static bool TryAssignConstructorArgs(
+            ConstructorInfo ctr,
+            int minArgCount,
+            List<(string Name, int? Index, object Value)> argList,
+            List<(string Name, int? Index, object Value)> indexedArgs,
+            List<(string Name, int? Index, object Value)> namedArgs,
+            [NotNullWhen(true)] out object[] ctrArgs
+         ) {
+            ctrArgs = null;
+            var ctrParams = ctr.GetParameters();
+            if (ctrParams.Length < minArgCount) {
+               return false;
+            }
+            var ctrArgList = new List<object>();
+            int remInputArgs = argList.Count;
+            for (int i = 0; i < ctrParams.Length; i++) {
+               var ctrParam = ctrParams[i];
+               var paramType = ctrParams[i].ParameterType;
+               if (indexedArgs.TryGetFirst(a => CanAssignInputArg(ctrParam, a), out var a)) {
+                  remInputArgs--;
+                  ctrArgList.Add(a.Value);
+               } else if (ctrParam.HasDefaultValue) {
+                  ctrArgList.Add(ctrParam.DefaultValue);
+               } else {
+                  return false;
+               }
+            }
+            if (remInputArgs > 0) {
+               return false;
+            }
+            Log.assert(ctrArgList.Count == ctrParams.Length);
+            ctrArgs = ctrArgList.ToArray();
+            return true;
+         }
+
+         static bool CanAssignInputArg(ParameterInfo pi, (string Name, int? Index, object Value) arg) {
+            if (!string.IsNullOrEmpty(arg.Name) && arg.Name != pi.Name) {
+               return false;
+            }
+            if (arg.Index.HasValue && arg.Index.Value != pi.Position) {
+               return false;
+            }
+            if (arg.Value is null) {
+               if (!pi.ParameterType.IsValueType && !pi.ParameterType.IsNullable()) {
+                  return false;
+               }
+            } else if (!pi.ParameterType.IsAssignableFrom(arg.Value.GetType())) {
+               return false;
+            }
+            return true;
+         }
       }
 
       /// <summary>

src/DataStreamSuite/DataStreamCore/DynamicLoading/PrimitiveManagerConfig.cs

@@ -0,0 +1,9 @@
+#nullable enable
+namespace Bts.DataStreamCore;
+
+using System.Collections.Generic;
+
+public class PrimitiveManagerConfig {
+   public List<string>? Assemblies { get; set; }
+   public List<string>? ExcludeTypes { get; set; }
+}

src/DataStreamSuite/DataStreamCore/IDataPacket.cs

@@ -9,6 +9,7 @@ namespace BCG
 {
    public interface IDataPacket
    {
+      IBlueTableSchema Schema { get; }
       //this is an ICollection of IDictionary
       void get_new_added_deleted(out ICollection new_rows, out ICollection added_rows, out ICollection deleted_rows);
    }
@@ -18,6 +19,8 @@ namespace BCG
       IBlueRowCollection<T> addedRows { get; }
       IBlueRowCollection<T> deletedRows { get; }
 
+      IBlueTableSchema IDataPacket.Schema => newRows.schema;
+
       void IDataPacket.get_new_added_deleted(out ICollection new_rows, out ICollection added_rows, out ICollection deleted_rows) {
          new_rows = newRows;
          added_rows = addedRows;

src/DataStreamSuite/DataStreamCore/Services/DataStreamCoreServices.cs

@@ -3,6 +3,9 @@ namespace Bts.DataStreamCore.Services;
 using BCG;
 using BCG.Utilities.General;
 
+using Bts.DataStream;
+
+using Microsoft.Extensions.Configuration;
 using Microsoft.Extensions.DependencyInjection;
 
 public static class DataStreamCoreServices {
@@ -14,4 +17,11 @@ public static class DataStreamCoreServices {
          ViewStreamDatabase.SetGlobalInstance(new ViewStreamDatabase(sp.GetRequiredService<ISubscriberCore>()));
          return ViewStreamDatabase.instance;
       });
+
+   public static IServiceCollection AddPrimitiveDataStreamFactory(this IServiceCollection services, IConfiguration configurationSection) =>
+      services.Configure<PrimitiveManagerConfig>(configurationSection)
+         .AddSingleton<IDataStreamFactory, PrimitiveManager>();
+
+   public static IServiceCollection AddCompositeDataStreamFactory(this IServiceCollection services) =>
+      services.AddSingleton<CompositeDataStreamFactory>();
 }

src/DataStreamSuite/DataStreamExpressions/Argument.cs

@@ -2,15 +2,40 @@ namespace Bts.DataStreamExpressions;
 
 public class Argument {
    public Argument() { }
-   public Argument(string str) => String = str;
-   public Argument(bool? boolean) => Boolean = boolean;
-   public Argument(int? integer) => Integer = integer;
-   public Argument(double? real) => Real = real;
-   public Argument(DateTime? dateTime) => DateTime = dateTime;
-   public Argument(DBNull? @null) => Null = @null;
-   public Argument(StatementExpression? statementExp) => StatementExp = statementExp;
-   public Argument(QuotedView? quotedView) => QuotedView = quotedView;
+   public Argument(string str, string? name = null) {
+      String = str;
+      Name = name;
+   }
+   public Argument(bool? boolean, string? name = null) {
+      Boolean = boolean;
+      Name = name;
+   }
+   public Argument(int? integer, string? name = null) {
+      Integer = integer;
+      Name = name;
+   }
+   public Argument(double? real, string? name = null) {
+      Real = real;
+      Name = name;
+   }
+   public Argument(DateTime? dateTime, string? name = null) {
+      DateTime = dateTime;
+      Name = name;
+   }
+   public Argument(DBNull? @null, string? name = null) {
+      Null = @null;
+      Name = name;
+   }
+   public Argument(StatementExpression? statementExp, string? name = null) {
+      StatementExp = statementExp;
+      Name = name;
+   }
+   public Argument(QuotedView? quotedView, string? name = null) {
+      QuotedView = quotedView;
+      Name = name;
+   }
 
+   public string? Name { get; }
    public string? String { get; }
    public bool? Boolean { get; }
    public int? Integer { get; }

src/DataStreamSuite/DataStreamExpressions/Assignment.cs

@@ -2,6 +2,8 @@ namespace Bts.DataStreamExpressions;
 
 using BCG;
 
+using Bts.DataStream;
+
 class Assignment : StatementExpression {
    readonly StatementExpression expression;
    readonly string id;
@@ -19,8 +21,8 @@ class Assignment : StatementExpression {
       return result;
    }
 
-   public override ViewStream? myGetView(IDictionary<string, StatementExpression> env) {
-      var result = expression.getView(env);
+   public override ViewStream? myGetView(IDictionary<string, StatementExpression> env, IDataStreamFactory streamFactory) {
+      var result = expression.getView(env, streamFactory);
       env[id] = expression;
       return result;
    }

src/DataStreamSuite/DataStreamExpressions/DataStreamExpression.cs

@@ -2,6 +2,8 @@ namespace Bts.DataStreamExpressions;
 
 using BCG;
 
+using Bts.DataStream;
+
 using System.Text;
 
 public class DataStreamExpression {
@@ -31,10 +33,10 @@ public class DataStreamExpression {
       return sb.ToString();
    }
 
-   public ViewStream? getView(IDictionary<string, StatementExpression> env) {
+   public ViewStream? getView(IDictionary<string, StatementExpression> env, IDataStreamFactory streamFactory) {
       for (int i = 0; i < Statements.Count - 1; i++) {
-         Statements[i].getView(env);
+         Statements[i].getView(env, streamFactory);
       }
-      return Statements[^1].getView(env);
+      return Statements[^1].getView(env, streamFactory);
    }
 }

src/DataStreamSuite/DataStreamExpressions/DataStreamExpressionGrammar.y

@@ -22,7 +22,7 @@
 }
 
 %token BOOLEAN_CONSTANT, INTEGER_CONSTANT, REAL_CONSTANT, DATE_TIME_CONSTANT, CS_STRING_CONSTANT, NULL_CONSTANT
-%token PARENTHESIS_OPEN, PARENTHESIS_CLOSE, COMMA, IDENTIFIER, EQUALS, QUOTE_OPEN, QUOTE_CLOSE
+%token PARENTHESIS_OPEN, PARENTHESIS_CLOSE, COMMA, COLON, IDENTIFIER, EQUALS, QUOTE_OPEN, QUOTE_CLOSE
 %token LINE_COMMENT
 
 %nonassoc EQUALS
@@ -72,13 +72,21 @@ Arguments :
 
 Argument :
    CS_STRING_CONSTANT { $$ = new($1.CsString); }
+   | IDENTIFIER COLON CS_STRING_CONSTANT { $$ = new($3.CsString, $1.Identifier); }
    | BOOLEAN_CONSTANT { $$ = new($1.Boolean); }
+   | IDENTIFIER COLON BOOLEAN_CONSTANT { $$ = new($3.Boolean, $1.Identifier); }
    | INTEGER_CONSTANT { $$ = new($1.Integer); }
+   | IDENTIFIER COLON INTEGER_CONSTANT { $$ = new($3.Integer, $1.Identifier); }
    | REAL_CONSTANT { $$ = new($1.Real); }
+   | IDENTIFIER COLON REAL_CONSTANT { $$ = new($3.Real, $1.Identifier); }
    | DATE_TIME_CONSTANT { $$ = new($1.DateTime); }
+   | IDENTIFIER COLON DATE_TIME_CONSTANT { $$ = new($3.DateTime, $1.Identifier); }
    | NULL_CONSTANT { $$ = new($1.Null); }
+   | IDENTIFIER COLON NULL_CONSTANT { $$ = new($3.Null, $1.Identifier); }
    | Statement { $$ = new($1); }
+   | IDENTIFIER COLON Statement { $$ = new($3, $1.Identifier); }
    | QuotedView { $$ = new($1); }
+   | IDENTIFIER COLON QuotedView { $$ = new($3, $1.Identifier); }
    ;
 
 QuotedView :

src/DataStreamSuite/DataStreamExpressions/DataStreamExpressionTokens.lex

@@ -31,6 +31,7 @@ Identifier               {Letter}({Letter}*[0-9]*_*)*
 OpenParenthesis          \(
 CloseParenthesis         \)
 Comma                    ,
+Colon                    :
 QuoteOpen                \[
 QuoteClose               \]
 
@@ -90,6 +91,10 @@ LineComment \/\/[^\n\r]*(\n|\r|\r\n)
    return (int)DataStreamExpressionToken.COMMA;
 }
 
+{Colon} {
+   return (int)DataStreamExpressionToken.COLON;
+}
+
 {QuoteOpen} {
    return (int)DataStreamExpressionToken.QUOTE_CLOSE;
 }

src/DataStreamSuite/DataStreamExpressions/PrimitiveExpression.cs

@@ -2,7 +2,8 @@ namespace Bts.DataStreamExpressions;
 
 using BCG;
 
-using System.Collections;
+using Bts.DataStream;
+
 using System.Collections.Immutable;
 
 public class PrimitiveExpression : StatementExpression {
@@ -20,7 +21,7 @@ public class PrimitiveExpression : StatementExpression {
    }
 
    public override string print(int startCol, IList<string> activeBindings) {
-      string prelude = "new " + PrimitiveManager.instance.find(myName).Name + " (";
+      string prelude = "new " + myName + " (";
 
       return prelude + myPrintArgs(startCol + prelude.Length, activeBindings) + ")";
    }
@@ -31,21 +32,19 @@ public class PrimitiveExpression : StatementExpression {
       return prelude + myPrintViewArgs(startCol + prelude.Length) + ")";
    }
 
-   public override ViewStream myGetView(IDictionary<string, StatementExpression> env) {
-      ArrayList resolvedArgs = new ArrayList();
+   public override ViewStream? myGetView(IDictionary<string, StatementExpression> env, IDataStreamFactory streamFactory) {
+      var resolvedArgs = myArgs.Select((arg, i) => ((string?)arg.Name, string.IsNullOrEmpty(arg.Name) ? (int?)i : null, ResolveArg(arg)));
+      return streamFactory.Create(myName, resolvedArgs);
 
-      foreach (var arg in myArgs) {
+      object? ResolveArg(Argument arg) {
          if (arg.StatementExp is not null) {
-            resolvedArgs.Add(arg.StatementExp.myGetView(env));
-            continue;
+            return arg.StatementExp.myGetView(env, streamFactory);
          }
          if (arg.QuotedView is not null) {
-            resolvedArgs.Add(arg.QuotedView.Value);
-            continue;
+            return arg.QuotedView.Value;
          }
-         resolvedArgs.Add(arg.Obj);
+         return arg.Obj;
       }
-      return PrimitiveManager.instance.build(myName, resolvedArgs);
    }
 
    string myPrintSingleArg(int atColumn, object? o, IList<string> activeBindings) {

src/DataStreamSuite/DataStreamExpressions/StatementExpression.cs

@@ -2,6 +2,8 @@ namespace Bts.DataStreamExpressions;
 
 using BCG;
 
+using Bts.DataStream;
+
 public abstract class StatementExpression {
    ViewStream? view;
 
@@ -13,10 +15,10 @@ public abstract class StatementExpression {
    public virtual string? printView(int startCol) =>
       null;
 
-   public ViewStream? getView(IDictionary<string, StatementExpression> env) =>
-      view ??= myGetView(env);
+   public ViewStream? getView(IDictionary<string, StatementExpression> env, IDataStreamFactory streamFactory) =>
+      view ??= myGetView(env, streamFactory);
 
-   public virtual ViewStream? myGetView(IDictionary<string, StatementExpression> env) =>
+   public virtual ViewStream? myGetView(IDictionary<string, StatementExpression> env, IDataStreamFactory streamFactory) =>
       null;
 
    public static string makeCSString(string s) {

src/DataStreamSuite/DataStreamExpressions/ViewNameExpression.cs

@@ -2,6 +2,8 @@ namespace Bts.DataStreamExpressions;
 
 using BCG;
 
+using Bts.DataStream;
+
 public class ViewNameExpression : StatementExpression {
    readonly string id;
 
@@ -14,6 +16,6 @@ public class ViewNameExpression : StatementExpression {
    public override string printView(int startCol) =>
       id;
 
-   public override ViewStream? myGetView(IDictionary<string, StatementExpression> env) =>
-      env.TryGetValue(id, out var exp) ? exp.getView(env) : new TableStream(id);
+   public override ViewStream? myGetView(IDictionary<string, StatementExpression> env, IDataStreamFactory streamFactory) =>
+      env.TryGetValue(id, out var exp) ? exp.getView(env, streamFactory) : new TableStream(id);
 }

src/DataStreamSuite/DataStreamPublisher/Dockerfile

@@ -1,8 +1,8 @@
 FROM bluetradingsystems/linux_dotnet7
 
-ARG CS_SOURCE_DIR="build/Release/bin/net7.0/linux-x64/publish"
-ARG CPP_SOURCE_DIR="install/data-stream-suite-linux"
-ARG CONF_FILE="src/DataStreamSuite/DataStreamPublisher/spark.conf"
+ARG DS_PUBLISHER_CS_SOURCE_DIR="build/Release/bin/net7.0/linux-x64/publish"
+ARG DS_PUBLISHER_CPP_SOURCE_DIR="install/data-stream-suite-linux"
+ARG DS_PUBLISHER_CONF_FILE="src/DataStreamSuite/DataStreamPublisher/spark.conf"
 
 ENV DOCKERIZE_VERSION=v0.7.0
 USER root
@@ -12,7 +12,7 @@ RUN apt-get update \
     && apt-get autoremove -yqq --purge wget && rm -rf /var/lib/apt/lists/*
 
 USER bts
-COPY --chown=bts:bts "${CPP_SOURCE_DIR}/lib" "bin"
-COPY --chown=bts:bts "${CS_SOURCE_DIR}" "bin"
-COPY --chown=bts:bts "${CONF_FILE}" ".local/share/bts/DataStreamPublisher/conf/spark.conf"
+COPY --chown=bts:bts "${DS_PUBLISHER_CPP_SOURCE_DIR}/lib" "bin"
+COPY --chown=bts:bts "${DS_PUBLISHER_CS_SOURCE_DIR}" "bin"
+COPY --chown=bts:bts "${DS_PUBLISHER_CONF_FILE}" ".local/share/bts/DataStreamPublisher/conf/spark.conf"
 WORKDIR "/home/bts/bin"

src/DataStreamSuite/DataStreamPublisher/Services/DataStreamPublisherService.cs

@@ -4,6 +4,7 @@ using BCG;
 
 using bts;
 
+using Bts.DataStream;
 using Bts.DataStreamCore;
 using Bts.DataStreamExpressions;
 using Bts.PubSub;
@@ -28,13 +29,28 @@ public class DataStreamPublisherService : IHostedService, IDisposable {
    readonly DataStreamPublisherConfig config;
    readonly IPubTableBroker pubTableBroker;
    readonly IWorkScheduler workScheduler;
+   readonly IDataStreamFactory streamFactory;
    List<IDisposable>? resources = new();
 
-   public DataStreamPublisherService(IPubTableBroker pubTableBroker, IWorkScheduler workScheduler, IOptions<DataStreamPublisherConfig> config)
-      : this(pubTableBroker, workScheduler, config.Value) { }
+   public DataStreamPublisherService(
+      IPubTableBroker pubTableBroker,
+      IEnumerable<IDataStreamFactory> streamFactories,
+      IWorkScheduler workScheduler,
+      IOptions<DataStreamPublisherConfig> config
+   )
+      : this(pubTableBroker, streamFactories, workScheduler, config.Value) { }
 
-   public DataStreamPublisherService(IPubTableBroker pubTableBroker, IWorkScheduler workScheduler, DataStreamPublisherConfig config) {
+   public DataStreamPublisherService(
+      IPubTableBroker pubTableBroker,
+      IEnumerable<IDataStreamFactory> streamFactories,
+      IWorkScheduler workScheduler,
+      DataStreamPublisherConfig config
+   ) {
       this.pubTableBroker = pubTableBroker;
+      var streamFactoryList = streamFactories.ToList();
+      streamFactory = streamFactoryList.Count == 1
+         ? streamFactoryList[0]
+         : new CompositeDataStreamFactory(streamFactoryList);
       this.workScheduler = workScheduler;
       this.config = config;
    }
@@ -48,7 +64,7 @@ public class DataStreamPublisherService : IHostedService, IDisposable {
                Log.Gen.warn("Skipping PublishedStream because it doesn't have a name (must define 'Name' in config json).");
                continue;
             }
-            var vs = CreateViewStream(publishedStream);
+            var vs = CreateViewStream(streamFactory, publishedStream);
             if (vs is null) {
                continue;
             }
@@ -72,7 +88,7 @@ public class DataStreamPublisherService : IHostedService, IDisposable {
       resources?.Clear();
    }
 
-   static ViewStream? CreateViewStream(PublishedStream ps) {
+   static ViewStream? CreateViewStream(IDataStreamFactory streamFactory, PublishedStream ps) {
       Log.Gen.info(
          $"Instantiating ViewPublisher for: {JsonSerializer.Serialize(ps)}");
       if (string.IsNullOrEmpty(ps.Name)) {
@@ -80,7 +96,7 @@ public class DataStreamPublisherService : IHostedService, IDisposable {
             $"Failed to instantiate ViewPublisher. Null/empty 'name': {JsonSerializer.Serialize(ps)}");
          return null;
       }
-      var vs = buildStream(ps.Expression);
+      var vs = buildStream(streamFactory, ps.Expression);
       if (vs is null) {
          Log.Gen.warn(
             $"Failed to instantiate ViewPublisher. Invalid 'expression': {JsonSerializer.Serialize(ps)}");
@@ -89,14 +105,14 @@ public class DataStreamPublisherService : IHostedService, IDisposable {
       return vs;
    }
 
-   static ViewStream? buildStream(string? streamExpression) {
+   static ViewStream? buildStream(IDataStreamFactory streamFactory, string? streamExpression) {
       if (streamExpression is null) {
          return null;
       }
       try {
          Log.Gen.info($"Building ViewStream: {streamExpression}");
          var exp = DataStreamExpressionsInterpreter.Evaluate(streamExpression);
-         return exp.getView(new Dictionary<string, StatementExpression>(0));
+         return exp.getView(new Dictionary<string, StatementExpression>(0), streamFactory);
       } catch (Exception ex) {
          Log.assert(false, $"Could not build stream. expression: {streamExpression}; ex: {ex}");
          return null;

src/DataStreamSuite/DataStreamPublisher/Services/DataStreamPublisherServices.cs

@@ -19,6 +19,7 @@ public static class DataStreamPublisherServices {
          .AddPubSubClient(configurationSection.GetSection("PubSub"))
          .AddPubTableBroker()
          .AddRemoteQueryManager()
+         .AddPrimitiveDataStreamFactory(configurationSection.GetSection("PrimitiveStream"))
          .Configure<DataStreamPublisherConfig>(configurationSection.GetSection("DataStreamPublisher"))
          .AddHostedService<DataStreamPublisherService>();
 }

src/DataStreamSuite/DataStreamPublisher/appsettings.json

@@ -12,6 +12,16 @@
   "ViewStreamDatabase": {
     "Interval": "00:00:00.1"
   },
+  "PrimitiveStream": {
+    "Assemblies": [
+      "DataStream.dll",
+      "SharedStreams.dll"
+    ],
+    "ExcludedTypes": [
+      "SettlementPriceStream",
+      "OHLCVStream"
+    ]
+  },
   "PubSub": {
     "HeartbeatInterval": 15000,
     "MissingHeartbeatsBeforeGc": 50,

src/DataStreamSuite/DataStreamSuite.sln

@@ -44,6 +44,8 @@ Project("{FAE04EC0-301F-11D3-BF4B-00C04F79EFBC}") = "Bts.Extensions.Hosting", "B
 EndProject
 Project("{FAE04EC0-301F-11D3-BF4B-00C04F79EFBC}") = "Bts.Net", "Bts.Net\Bts.Net.csproj", "{D92E1456-91C1-43FA-AA87-67BFE663944F}"
 EndProject
+Project("{FAE04EC0-301F-11D3-BF4B-00C04F79EFBC}") = "PandasExpressions", "PandasExpressions\PandasExpressions.csproj", "{32F45D7C-6DE5-4660-AD45-EDEE3B41E26F}"
+EndProject
 Global
 	GlobalSection(SolutionConfigurationPlatforms) = preSolution
 		Debug|Any CPU = Debug|Any CPU
@@ -137,5 +139,9 @@ Global
 		{D92E1456-91C1-43FA-AA87-67BFE663944F}.Debug|Any CPU.Build.0 = Debug|Any CPU
 		{D92E1456-91C1-43FA-AA87-67BFE663944F}.Release|Any CPU.ActiveCfg = Release|Any CPU
 		{D92E1456-91C1-43FA-AA87-67BFE663944F}.Release|Any CPU.Build.0 = Release|Any CPU
+		{32F45D7C-6DE5-4660-AD45-EDEE3B41E26F}.Debug|Any CPU.ActiveCfg = Debug|Any CPU
+		{32F45D7C-6DE5-4660-AD45-EDEE3B41E26F}.Debug|Any CPU.Build.0 = Debug|Any CPU
+		{32F45D7C-6DE5-4660-AD45-EDEE3B41E26F}.Release|Any CPU.ActiveCfg = Release|Any CPU
+		{32F45D7C-6DE5-4660-AD45-EDEE3B41E26F}.Release|Any CPU.Build.0 = Release|Any CPU
 	EndGlobalSection
 EndGlobal

src/DataStreamSuite/DataStreamTests/BlueColumnTableTests.cs

@@ -0,0 +1,114 @@
+namespace Bts.DataStreamTests;
+
+using BCG;
+
+using Bts.DataStreamCore;
+
+using Spark.Core;
+
+[TestFixture]
+public class BlueColumnTableTests {
+   static IBlueTableSchema defaultSchema = new BlueTableSchemaBuilder("addTestTable") {
+         ("id", typeof(int)),
+         ("name", typeof(string)),
+         ("value", typeof(double)),
+      }
+      .ToSchema();
+
+   static BlueColumnTable getTable(params Variant[][] rows) {
+      var bct = new BlueColumnTable(defaultSchema);
+      for (int i = 0; i < rows.Length; i++) {
+         bct.Add(rows[i]);
+      }
+      return bct;
+   }
+
+   [Test]
+   public void TestAdd() {
+      var rows = new Variant[][] {
+         new Variant[] { 1, "Joe", 32 },
+         new Variant[] { 2, "Angie", 26.5 },
+      };
+      var bct = getTable();
+      Is.EqualTo(bct.Count == 0);
+      bct.Add(rows[0]);
+      bct.Add(rows[1]);
+      Is.EqualTo(bct.Count == 2);
+      assertRowEquals(bct[rows[0][0].GetInt32()], rows[0]);
+      assertRowEquals(bct[rows[1][0].GetInt32()], rows[1]);
+      assertRowSequenceEqual(bct, rows);
+      rows[0] = new Variant[] { 1, "Mika", 4.2 };
+      bct.AddOrUpdate(rows[0]);
+      Is.EqualTo(bct.Count == 2);
+      assertRowEquals(bct[rows[0][0].GetInt32()], rows[0]);
+   }
+
+   [Test]
+   public void TestRemove() {
+      var rows = new Variant[][] {
+         new Variant[] { 5, "Angie", 26.5 },
+         new Variant[] { 1, "Joe", 32 },
+         new Variant[] { 8, "Myrtle", 103 },
+      };
+      var bct = getTable(rows);
+      bct.Remove(rows[1][0].GetInt32());
+      assertRowSequenceEqual(bct, new[] { rows[0], rows[2] });
+   }
+
+   [Test]
+   public void TestRemoveReAdd() {
+      var rows = new Variant[][] {
+         new Variant[] { 5, "Angie", 26.5 },
+         new Variant[] { 1, "Joe", 32 },
+         new Variant[] { 8, "Myrtle", 103 },
+      };
+      var bct = getTable(rows);
+      bct.Remove(rows[1][0].GetInt32());
+      bct.Add(rows[1]);
+      assertRowSequenceEqual(bct, rows);
+   }
+
+   [Test]
+   public void TestClear() {
+      var rows = new Variant[][] {
+         new Variant[] { 5, "Angie", 26.5 },
+         new Variant[] { 1, "Joe", 32 },
+         new Variant[] { 8, "Myrtle", 103 },
+      };
+      var bct = getTable(rows);
+      bct.Clear();
+      assertRowSequenceEqual(bct, Array.Empty<Variant[]>());
+   }
+
+   static void assertRowEquals(IBlueRow row, Variant[] expectedValues, IBlueTableSchema? schema = null) {
+      schema ??= defaultSchema;
+      for (int i = 0; i < schema.columnInfo.Length; i++) {
+         Is.EqualTo(Equals(row[schema.columnInfo[i].colNo], expectedValues[i]));
+         if (!schema.columnInfo.TryGetFirst(c => c!.colNo == i, out var col)) {
+            throw new InvalidOperationException($"Schema has no column at index {i}");
+         }
+         Is.EqualTo(Equals(row[col.colName], expectedValues[i]));
+      }
+   }
+
+   static void assertRowSequenceEqual(BlueColumnTable bct, IEnumerable<Variant[]> rowValues) {
+      if (!bct.schema.columnInfo.TryGetFirst(c => c!.colName == "id", out var idCol)) {
+         throw new ArgumentException("Schema has no id column", nameof(bct));
+      }
+      Assert.That(bct.Cast<IBlueRow>().SequenceEqual(
+         rowValues.OrderBy(r => r[idCol.colNo].GetInt32()).Select(r => new MyRow(r, bct.schema)),
+         BlueRowEqualityComparer.Instance));
+   }
+
+   class MyRow : IBlueRow {
+      readonly Variant[] values;
+
+      public MyRow(Variant[] values, IBlueTableSchema schema) {
+         this.values = values;
+         this.schema = schema;
+      }
+
+      public IBlueTableSchema schema { get; }
+      public object? this[int index] => values[index].GetValue();
+   }
+}

src/DataStreamSuite/Directory.Build.targets

@@ -0,0 +1,18 @@
+<Project>
+  
+  <Import Project="..\Directory.Build.targets" />
+
+  <Target
+    Name="GenerateRowTypes"
+    BeforeTargets="BeforeCompile"
+    Condition="'@(RowType)'!=''"
+    Inputs="%(RowType.SchemaJson);%(RowType.Identity)"
+    Outputs="$(IntermediateOutputPath)/RowTypes/%(RowType.TypeName).cs">
+    <MakeDir Directories="$(IntermediateOutputPath)/RowTypes" />
+    <Exec Command="dotnet t4 %(RowType.Identity) -p SchemaJson=%(RowType.SchemaJson) -p TypeName=%(RowType.TypeName) -p TableName=%(RowType.TableName) -o $(IntermediateOutputPath)/RowTypes/%(RowType.TypeName).cs" />
+    <ItemGroup>
+      <Compile Include="$(IntermediateOutputPath)/RowTypes/%(RowType.TypeName).cs" />
+    </ItemGroup>
+  </Target>
+
+</Project>

src/DataStreamSuite/PandasExpressions/Class1.cs

@@ -0,0 +1,6 @@
+namespace PandasExpressions;
+
+public class Class1
+{
+
+}

src/DataStreamSuite/PandasExpressions/PandasExpressionGrammar.y

@@ -0,0 +1,72 @@
+%namespace Bts.DataStreamExpressions
+%partial
+%visibility internal
+%tokentype DataStreamExpressionToken
+%parsertype DataStreamExpressionParser
+
+%union {
+   public int Integer;
+   public double Real;
+   public bool Boolean;
+   public None None;
+   public string Identifier;
+   public DataFrameExpression DataFrame;
+   public StatementExpression Statement;
+   public PrimitiveExpression Primitive;
+   public ImmutableList<Argument> Arguments;
+   public Argument Argument;
+}
+
+%token BOOLEAN_CONSTANT, INTEGER_CONSTANT, REAL_CONSTANT, NONE_CONSTANT
+%token PARENTHESIS_OPEN, PARENTHESIS_CLOSE, BRACKET_OPEN, BRACKET_CLOSE, DOT, COMMA, COLON, IDENTIFIER, EQUALS
+%token AND, OR, PLUS, MINUS, TIMES, DIVIDE
+
+%nonassoc EQUALS
+
+%type <DataFrame> DataFrame
+%type <Statement> Statement
+%type <Argument> Argument
+%type <Arguments> Arguments
+
+%%
+
+ArithmeticColumnExpression :
+
+DataFrameExpression :
+   IDENTIFIER O
+
+Arguments :
+   {
+      $$ = System.Collections.Immutable.ImmutableList<Argument>.Empty;
+   }
+   | Argument {
+      $$ = System.Collections.Immutable.ImmutableList<Argument>.Empty.Add($1);
+   }
+   | Arguments COMMA Argument {
+      $$ = $1.Add($3);
+   }
+   ;
+
+ColumnExpression :
+   IDENTIFIER DOT IDENTIFIER {
+      $$ = new ColumnNameExpression($1, $3);
+   }
+   | IDENTIFIER OPEN_BRACKET IDENTIFIER CLOSE_BRACKET {
+      $$ = new ColumnNameExpression($1, $3);
+   }
+   | ColumnExpression PLUS ColumnExpression {
+      $$ = $1.Plus($3);
+   }
+   | ColumnExpression MINUS ColumnExpression {
+      $$ = $1.Minus($3);
+   }
+   | ColumnExpression TIMES ColumnExpression {
+      $$ = $1.Multiply($3);
+   }
+   | ColumnExpression DIVIDE ColumnExpression {
+         
+   }
+   | ColumnExpression AND ColumnExpression {
+         
+   }
+   | ColumnExpression OR ColumnExpression

src/DataStreamSuite/PandasExpressions/PandasExpressionTokens.lex

@@ -0,0 +1,93 @@
+%namespace Bts.PandasExpressions
+%scannertype PandasExpressionScanner
+%visibility internal
+%tokentype PandasExpressionToken
+
+%option stack, minimize, parser, verbose, persistbuffer, noembedbuffers
+
+Identifier               [a-zA-Z_][a-zA-Z_0-9]*
+Equals                   =
+IntegerConstant          -?[0-9]+
+RealConstant             -?[0-9]+\.[0-9]+
+BooleanConstant          True|False
+And                      &&
+Or                       ||
+None                     None
+OpenParenthesis          \(
+CloseParenthesis         \)
+OpenBracket              \[
+CloseBracket             \]
+Dot                      .
+Comma                    ,
+Colon                    :
+Plus                     +
+Minus                    -
+Times                    *
+Divide                   /
+
+
+%%
+
+{Equals} {
+   return (int)PandasExpressionToken.EQUALS;
+}
+
+{BooleanConstant} {
+   yylval.Boolean = yytext.ToLower() switch {
+      "True" => true,
+      "False" => false,
+      _ => throw new Exception(
+         string.Format("Expected yytext to be 'True' or 'False' (actually '{0}')", yytext))
+   };
+   return (int)PandasExpressionToken.BOOLEAN_CONSTANT;
+}
+
+{IntegerConstant} {
+   yylval.Integer = int.Parse(yytext);
+   return (int)PandasExpressionToken.INTEGER_CONSTANT;
+}
+
+{RealConstant} {
+   yylval.Real = double.Parse(yytext);
+   return (int)PandasExpressionToken.REAL_CONSTANT;
+}
+
+{None} {
+   yylval.None = None.Value;
+   return (int)PandasExpressionToken.NONE_CONSTANT;
+}
+
+{OpenParenthesis} {
+   return (int)PandasExpressionToken.PARENTHESIS_OPEN;
+}
+
+{CloseParenthesis} {
+   return (int)PandasExpressionToken.PARENTHESIS_CLOSE;
+}
+
+{Dot} {
+   return (int)PandasExpressionToken.DOT;
+}
+
+{Comma} {
+   return (int)PandasExpressionToken.COMMA;
+}
+
+{Colon} {
+   return (int)PandasExpressionToken.COLON;
+}
+
+{OpenBracket} {
+   return (int)PandasExpressionToken.BRACKET_OPEN;
+}
+
+{CloseBracket} {
+   return (int)PandasExpressionToken.BRACKET_CLOSE;
+}
+
+{Identifier} {
+   yylval.Identifier = yytext;
+   return (int)PandasExpressionToken.IDENTIFIER;
+}
+
+%%

src/DataStreamSuite/PandasExpressions/PandasExpressions.csproj

@@ -0,0 +1,9 @@
+<Project Sdk="Microsoft.NET.Sdk">
+
+  <PropertyGroup>
+    <TargetFramework>net7.0</TargetFramework>
+    <ImplicitUsings>enable</ImplicitUsings>
+    <Nullable>enable</Nullable>
+  </PropertyGroup>
+
+</Project>

src/DataStreamSuite/PubSub/ManagedTCP/AsyncTCPConnection_AsyncWrapper.cs

@@ -182,9 +182,9 @@ namespace bts
             }
             n_transfers++;
             total_bytes_read += bytes_read;
-            Log.Gen.info($"AsyncTCPConnection_AsyncWrapper._read_fixed_size_msg n_bytes: {n_bytes} bytes_read: {bytes_read} total_bytes_read: {total_bytes_read} buff_len: {buffer.Length}");
+            Log.DataFlow.diag($"AsyncTCPConnection_AsyncWrapper._read_fixed_size_msg n_bytes: {n_bytes} bytes_read: {bytes_read} total_bytes_read: {total_bytes_read} buff_len: {buffer.Length}");
          }
-         Log.Gen.info($"AsyncTCPConnection_AsyncWrapper._read_fixed_size_msg read {n_bytes} bytes in {n_transfers} transfers");
+         Log.DataFlow.diag($"AsyncTCPConnection_AsyncWrapper._read_fixed_size_msg read {n_bytes} bytes in {n_transfers} transfers");
          return buffer;
       }
 
@@ -223,13 +223,13 @@ namespace bts
          BinaryReader br = new BinaryReader(new MemoryStream(sz_arr));
          br.BaseStream.Position = 0;
          int msg_sz = br.ReadInt32();
-         Log.Gen.info($"AsyncTCPConnection_AsyncWrapper._read_one_message read msg_sz: {msg_sz}");
+         Log.DataFlow.diag($"AsyncTCPConnection_AsyncWrapper._read_one_message read msg_sz: {msg_sz}");
          if (msg_sz < 0)
          {
             throw new IOException("Read 0 bytes, the connection was closed before all data could be read.");
          }
          byte[] msg = _read_fixed_size_msg(msg_sz);
-         Log.Gen.info($"AsyncTCPConnection_AsyncWrapper._read_one_message read msg sz: {msg.Length}");
+         Log.DataFlow.diag($"AsyncTCPConnection_AsyncWrapper._read_one_message read msg sz: {msg.Length}");
          // TODO, read first 4 bytes and decide hb or data
          OneRecvMessage rv = new OneRecvMessage(msg);
          return rv;
@@ -336,4 +336,4 @@ namespace bts
          return _connection_id.ToString();
       }
    }
-}
+}

src/DataStreamSuite/PubSub/Server/SingleTagData.cs

@@ -19,6 +19,9 @@ namespace bts {
    /// </summary>
    public class SingleTagData {
       private readonly IPubSubServerTagHandler? _tag_handler;
+      readonly HashSet<string> noisyTags = new() {
+         "pss_hb"
+      };
 
       public SingleTagData(string tag, string tag_type, IPubSubServerTagHandler tag_handler) {
          bts.Log.Gen.info($"SingleTagData constructed for tag: {tag} tag_type: {tag_type}");
@@ -36,7 +39,9 @@ namespace bts {
       // PubsubState calls this
       public void handle_publish(PublishMsg pub_msg) {
          string handler_str = _tag_handler != null ? _tag_handler.GetType().ToString() : "NULL";
-         bts.Log.Gen.info($"SingleTagData.handle_publish tag: {tag} pub_msg: {pub_msg} handler: {handler_str} ");
+         if (pub_msg.tag is not null && !noisyTags.Contains(pub_msg.tag)) {
+            bts.Log.Gen.info($"SingleTagData.handle_publish tag: {tag} pub_msg: {pub_msg} handler: {handler_str} ");
+         }
          try {
             // tag_handler may be null. If it throws (a null reference ex), we'll catch and handle
             // it like any other ex.

src/DataStreamSuite/SharedStreams/Business/MarketData/OHLCVStream.cs

@@ -111,7 +111,14 @@ namespace bts.SilverPine {
       :
       this(symbol, Enum.Parse<OHLCVBarSize>(bar_size)) { }
 
-
-
+      internal OHLCVStream(ViewStream source, OHLCVBarSize bar_size) {
+         ViewStream rs = source;
+         rs = new SchemaGuesserStream(rs);
+         if (bar_size == OHLCVBarSize.ONE_DAY) {
+            rs = new DateStandardizationStream(rs, "ts_event", DateConversion.ChopTime);
+         }
+         rs = new NewProjectStream(rs, "symbol:contract");
+         mySetRealSource(rs);
+      }
    }
-}
+}

src/DataStreamSuite/SharedStreams/Business/MarketData/SettlementPriceStream.cs

@@ -50,4 +50,4 @@ namespace bts.SilverPine {
       }
 
    }
-}
+}

src/DataStreamSuite/SharedStreams/NL/DatabaseConfig.cs

@@ -0,0 +1,6 @@
+namespace bts.SilverPine;
+
+public class DatabaseConfig {
+   public string? Path { get; set; }
+   public string? Query { get; set; }
+}

src/DataStreamSuite/SharedStreams/NL/NLPromptConfig.cs

@@ -0,0 +1,77 @@
+namespace bts.SilverPine;
+
+using Spark.Core.Json;
+
+using System.Collections.Generic;
+using System.Text.Json;
+using System.Text.Json.Serialization;
+
+public class NLPromptConfig {
+   static NLPromptConfig? instance;
+
+   public static NLPromptConfig Instance =>
+      instance ?? throw new InvalidOperationException(
+         "SetGlobalConfig must be called to initialize config static singleton");
+
+   public List<FewShotPrompt> prompts { get; } = new(0);
+
+   public static void SetGlobalConfig(NLPromptConfig cfg) {
+      // This should only be called once
+      Log.isnull(instance);
+      instance = Log.notnull(cfg);
+   }
+
+   public static NLPromptConfig Default() => new();
+
+   // Represents a pair of a natural language query and its dsl
+   public class FewShotExample {
+      public string? natural_language { get; set; }
+      public string? dsl { get; set; }
+   }
+
+   public class FewShotPrompt {
+      public string? category { get; set; }
+
+      public string? description { get; set; }
+
+      public bool IsEnabled { get; set; } = true;
+
+      // MultiLineDescription is simply here for convenience. It is functionally redundant, as it
+      // serves the same purpose as 'description'. If 'description' is too long, you can break it
+      // up into an array of strings and assign it to MultiLineDescription instead.
+      public List<string>? MultiLineDescription { get; set; }
+
+      // the actual prompt given by the user/CME
+      public string? prompt { get; set; }
+
+      public List<FewShotExample> examples { get; } = new(0);
+
+      public override string ToString() {
+         var desc = description ?? string.Concat(MultiLineDescription.empty_if_null());
+         string rv = $"System: {desc}\n";
+         int ex_no = 0;
+         foreach (FewShotExample ex in examples ?? new(0)) {
+            rv += $"{++ex_no}. Natural Language: \"{ex.natural_language}\"" + "\n";
+            rv += $"    DSL: {ex.dsl}\n";
+
+         }
+         return rv;
+      }
+
+      // Reads normal strings and arrays of lines
+      public class FewShotPromptLinesConverter : JsonConverter<string> {
+         public override string? Read(ref Utf8JsonReader reader, Type typeToConvert, JsonSerializerOptions options) {
+            if (reader.ReadString(out var str)) {
+               return str;
+            }
+            if (reader.ReadArray(out var array)) {
+               return string.Join('\n', array.Select(v => v?.AsString()));
+            }
+            throw new JsonException("Failed to parse FewShotPrompt lines");
+         }
+
+         public override void Write(Utf8JsonWriter writer, string value, JsonSerializerOptions options) =>
+            writer.WriteStringValue(value);
+      }
+   }
+}

src/DataStreamSuite/SharedStreams/NL/NLStreamFactory.cs

@@ -0,0 +1,42 @@
+namespace bts.SilverPine;
+
+using BCG;
+
+using Bts.DataStream;
+
+using mbts.utils;
+
+using System.Collections.Generic;
+
+public class NLStreamFactory : IRecursiveDataStreamFactory {
+   IDataStreamFactory? compositeFactory;
+
+   public ViewStream? Create(string streamId, IEnumerable<(string? Name, int? Index, object? Value)> args) {
+      if (SupportedStreams.NaturalLanguage.IsMatch(streamId)) {
+         if (compositeFactory is null) {
+            Log.DataFlow.warn("Failed to create NLStream because 'compositeFactory' is null.");
+            return null;
+         }
+         if (args.SingleOrDefault().Value is string prompt) {
+            return new NLStream(prompt, compositeFactory);
+         }
+      }
+      return null;
+   }
+
+   public IEnumerable<IDataStreamMetadata> GetSupportedStreams() {
+      yield return SupportedStreams.NaturalLanguage;
+   }
+
+   public void OnCompositeFactory(IDataStreamFactory compositeFactory) =>
+      this.compositeFactory = compositeFactory;
+
+   static class SupportedStreams {
+      public static readonly DataStreamMetadata NaturalLanguage = new() {
+         Name = "nl",
+         AlternateNames = new() { "naturalLanguage" }
+      };
+
+      public static string NlName => ExceptionAssert.not_null(NaturalLanguage.Name);
+   }
+}

src/DataStreamSuite/SharedStreams/NL/NLToDSLStream.cs

@@ -40,4 +40,4 @@ namespace bts.SilverPine {
 
 
    }
-}
+}

src/DataStreamSuite/SharedStreams/NL/NLToIndexedFewShotPromptStream.cs

@@ -7,38 +7,16 @@ namespace bts.SilverPine {
 
    using System.Text.Json;
 
+   using Root = NLPromptConfig;
+   using FewShotExample = NLPromptConfig.FewShotExample;
+   using FewShotPrompt = NLPromptConfig.FewShotPrompt;
 
    [DataStreamUserRespresentation("nl_to_indexed_fewshot_prompt")]
    // given an nl prompt, retreives the correct fewshot prompt and converts it into a form
    // that be sent to openAI as a prompt
    public class NLToIndexedFewShotPromptStream : ViewStream {
 
-      // Represents a pair of a natural language query and its dsl
-      public class FewShotExample {
-         public string natural_language { get; set; }
-         public string dsl { get; set; }
-      }
-
       // Class to represent each prompt category
-      public class FewShotPrompt {
-         public string category { get; set; }
-         // this is the explaination of the DSL
-         public string description { get; set; }
-         // th actual prompt given by the user/CME
-         public string prompt { get; set; }
-         public List<FewShotExample> examples { get; set; }
-
-         public override string ToString() {
-            string rv = $"System: {description}\n";
-            int ex_no = 0;
-            foreach (FewShotExample ex in examples) {
-               rv += $"{++ex_no}. Natural Language: \"{ex.natural_language}\"" + "\n";
-               rv += $"    DSL: {ex.dsl}\n";
-
-            }
-            return rv;
-         }
-      }
 
       // this is how it should be prepended to the actual prompt:
       //   System: This is a few-shot learning example for converting natural language prompts to a DSL function for settlement data retrieval. The DSL function can take either a single date or a date range. Assume today's date is 10/11/2024. The function is structured as follows:
@@ -66,21 +44,6 @@ namespace bts.SilverPine {
 
       // [Your Real Prompt Goes Here]
 
-
-      // Root class to map the JSON structure
-      public class Root {
-         public List<FewShotPrompt> prompts { get; set; }
-      }
-
-      public static Root read_json_file() {
-         // TODO
-         string json_file_path = @"C:\Users\m_tah\development\spark\src\DataStreamSuite\SharedStreams\NL\FewShotPrompts.json";
-         string json_string = File.ReadAllText(json_file_path);
-         var options = new JsonSerializerOptions { PropertyNameCaseInsensitive = true };
-         Root root = JsonSerializer.Deserialize<Root>(json_string, options);
-         return root;
-      }
-
       private readonly string _nl_prompt;
       private string _fewshot_index_prompt;
       private string _fewshot_index_reponse;
@@ -101,7 +64,7 @@ namespace bts.SilverPine {
       }
 
       protected override void myOnFirstSubscriber() {
-         Root root = read_json_file();
+         var root = NLPromptConfig.Instance;
          string meta_prompt = build_fewshot_index_prompt(root, _nl_prompt);
          _fewshot_index_prompt = meta_prompt;
          Log.Gen.info($"NLToDSLStream.myOnFirstSubscriber _fewshot_index_prompt: {_fewshot_index_prompt}");
@@ -138,7 +101,7 @@ namespace bts.SilverPine {
             string[] toks = response.Split('.');
             index = Int32.Parse(toks[0]);
          }
-         var root = read_json_file();
+         var root = NLPromptConfig.Instance;
          FewShotPrompt fs_prompt = root.prompts[index];
          string final_prompt = _build_fewshot_prompt(fs_prompt, _nl_prompt);
          _fire_event(final_prompt);
@@ -166,6 +129,10 @@ namespace bts.SilverPine {
          string meta_promp = "Take a look at the following prompt examples:\n";
          int rv = 0;
          foreach (FewShotPrompt fs_entry in root.prompts) {
+            if (!fs_entry.IsEnabled) {
+               continue;
+            }
+            var prompt = fs_entry.prompt;
             meta_promp += $"{rv++}. {fs_entry.prompt}\n";
          }
          meta_promp += "Based on the above prompts, tell me which prompt (give me answer as the index only, and nothing else) does the following prompt most closely matches to: \n";
@@ -194,4 +161,4 @@ namespace bts.SilverPine {
          return rv;
       }
    }
-}
+}

src/DataStreamSuite/SharedStreams/NL/OhlcvStreamConfig.cs

@@ -0,0 +1,32 @@
+namespace bts.SilverPine;
+
+// public class DateRange {
+//    public DateTime Begin { get; set; }
+//    public DateTime End { get; set; }
+// }
+
+public class OhlcvDatabaseConfig {
+   public class DatasetConfig {
+      public string? Path { get; set; }
+      public string? Query { get; set; }
+      public List<string>? QueryParams { get; set; }
+   }
+
+   public DatasetConfig? Bbo1s { get; set; }
+   public DatasetConfig? Bbo1m { get; set; }
+   public DatasetConfig? Ohlcv1s { get; set; }
+   public DatasetConfig? Ohlcv1m { get; set; }
+   public DatasetConfig? Ohlcv1h { get; set; }
+   public DatasetConfig? Ohlcv1d { get; set; }
+   public DatasetConfig? Mbo { get; set; }
+
+   // public List<string>? Symbols { get; set; }
+   // public List<string>? Venues { get; set; }
+   // public List<string>? Types { get; set; }
+   // public List<DateRange>? DateRanges { get; set; }
+   // public string? Path { get; set; }
+}
+
+public class OhlcvStreamConfig {
+   public OhlcvDatabaseConfig? Database { get; set; }
+}

src/DataStreamSuite/SharedStreams/NL/OpenAIPromptStream.cs

@@ -103,6 +103,10 @@ namespace bts.SilverPine {
                                              Tuple.Create("id", TypeCode.Int32),
                                              Tuple.Create("prompt", TypeCode.String),
                                              Tuple.Create("response", TypeCode.String));
+            const string DSL_PREFIX = "DSL: ";
+            if (response.StartsWith(DSL_PREFIX)) {
+               response = response.Remove(0, DSL_PREFIX.Length);
+            }
             IBlueRow nr = new BlueRow(schema, 0, _prompt, response);
             BlueTable nt = new BlueTable(schema);
             nt.addRow(nr);

src/DataStreamSuite/SharedStreams/RowTemplates/AvgOrderLatencySchema.json

@@ -0,0 +1,34 @@
+{
+  "IsTimeSeries": true,
+  "Columns": [
+    {
+      "Name": "Id",
+      "Type": "int",
+      "Key": "id"
+    },
+    {
+      "Name": "Timestamp",
+      "Type": "DateTime"
+    },
+    {
+      "Name": "AvgLatency",
+      "Type": "TimeSpan"
+    },
+    {
+      "Name": "Venue",
+      "Type": "string"
+    },
+    {
+      "Name": "Symbol",
+      "Type": "string"
+    },
+    {
+      "Name": "OrderCount",
+      "Type": "int"
+    },
+    {
+      "Name": "Volume",
+      "Type": "long"
+    }
+  ]
+}

src/DataStreamSuite/SharedStreams/RowTemplates/BboSchema.json

@@ -0,0 +1,38 @@
+{
+  "IsTimeSeries": true,
+  "Columns": [
+    {
+      "Name": "Id",
+      "Type": "int",
+      "Key": "id"
+    },
+    {
+      "Name": "Timestamp",
+      "Type": "DateTime"
+    },
+    {
+      "Name": "Venue",
+      "Type": "string"
+    },
+    {
+      "Name": "Symbol",
+      "Type": "string"
+    },
+    {
+      "Name": "BidPrice",
+      "Type": "double"
+    },
+    {
+      "Name": "BidQuantity",
+      "Type": "double"
+    },
+    {
+      "Name": "AskPrice",
+      "Type": "double"
+    },
+    {
+      "Name": "AskQuantity",
+      "Type": "double"
+    }
+  ]
+}

src/DataStreamSuite/SharedStreams/RowTemplates/MboSchema.json

@@ -0,0 +1,54 @@
+{
+  "IsTimeSeries": true,
+  "Columns": [
+    {
+      "Name": "Id",
+      "Type": "int",
+      "Key": "id"
+    },
+    {
+      "Name": "SendTimestamp",
+      "Type": "DateTime"
+    },
+    {
+      "Name": "Timestamp",
+      "Type": "DateTime"
+    },
+    {
+      "Name": "SequenceNumber",
+      "Type": "int"
+    },
+    {
+      "Name": "Venue",
+      "Type": "string"
+    },
+    {
+      "Name": "Symbol",
+      "Type": "string"
+    },
+    {
+      "Name": "Contract",
+      "Type": "string"
+    },
+    {
+      "Name": "Side",
+      "Type": "bts.api.Side"
+    },
+    {
+      "Name": "Price",
+      "Type": "double"
+    },
+    {
+      "Name": "Quantity",
+      "Type": "double"
+    },
+    {
+      "Name": "OrderId",
+      "Type": "int"
+    },
+    {
+      "Name": "Action",
+      "Type": "string"
+    }
+  ]
+}

src/DataStreamSuite/SharedStreams/RowTemplates/MbpSchema.json

@@ -0,0 +1,170 @@
+{
+  "IsTimeSeries": true,
+  "Columns": [
+    {
+      "Name": "Id",
+      "Type": "int",
+      "Key": "id"
+    },
+    {
+      "Name": "Timestamp",
+      "Type": "DateTime"
+    },
+    {
+      "Name": "Venue",
+      "Type": "string"
+    },
+    {
+      "Name": "Symbol",
+      "Type": "string"
+    },
+    {
+      "Name": "BidPrices",
+      "Key": null,
+      "Type": "double[]",
+      "Value": "{ get; } = new double[5];",
+      "Equals": "BidPrices.SequenceEqual(other.BidPrices)"
+    },
+    {
+      "Name": "BidQuantities",
+      "Key": null,
+      "Type": "double[]",
+      "Value": "{ get; } = new double[5];",
+      "Equals": "BidQuantities.SequenceEqual(other.BidQuantities)"
+    },
+    {
+      "Name": "BidPrice1",
+      "Type": "double",
+      "Value": "BidPrices[0]",
+      "Equals": null
+    },
+    {
+      "Name": "BidQuantity1",
+      "Type": "double",
+      "Value": "BidQuantities[0]",
+      "Equals": null
+    },
+    {
+      "Name": "BidPrice2",
+      "Type": "double",
+      "Value": "BidPrices[1]",
+      "Equals": null
+    },
+    {
+      "Name": "BidQuantity2",
+      "Type": "double",
+      "Value": "BidQuantities[1]",
+      "Equals": null
+    },
+    {
+      "Name": "BidPrice3",
+      "Type": "double",
+      "Value": "BidPrices[2]",
+      "Equals": null
+    },
+    {
+      "Name": "BidQuantity3",
+      "Type": "double",
+      "Value": "BidQuantities[2]",
+      "Equals": null
+    },
+    {
+      "Name": "BidPrice4",
+      "Type": "double",
+      "Value": "BidPrices[3]",
+      "Equals": null
+    },
+    {
+      "Name": "BidQuantity4",
+      "Type": "double",
+      "Value": "BidQuantities[3]",
+      "Equals": null
+    },
+    {
+      "Name": "BidPrice5",
+      "Type": "double",
+      "Value": "BidPrices[4]",
+      "Equals": null
+    },
+    {
+      "Name": "BidQuantity5",
+      "Type": "double",
+      "Value": "BidQuantities[4]",
+      "Equals": null
+    },
+    {
+      "Name": "AskPrices",
+      "Key": null,
+      "Type": "double[]",
+      "Value": "{ get; } = new double[5];",
+      "Equals": "AskPrices.SequenceEqual(other.AskPrices)"
+    },
+    {
+      "Name": "AskQuantities",
+      "Key": null,
+      "Type": "double[]",
+      "Value": "{ get; } = new double[5];",
+      "Equals": "AskQuantities.SequenceEqual(other.AskQuantities)"
+    },
+    {
+      "Name": "AskPrice1",
+      "Type": "double",
+      "Value": "AskPrices[0]",
+      "Equals": null
+    },
+    {
+      "Name": "AskQuantity1",
+      "Type": "double",
+      "Value": "AskQuantities[0]",
+      "Equals": null
+    },
+    {
+      "Name": "AskPrice2",
+      "Type": "double",
+      "Value": "AskPrices[1]",
+      "Equals": null
+    },
+    {
+      "Name": "AskQuantity2",
+      "Type": "double",
+      "Value": "AskQuantities[1]",
+      "Equals": null
+    },
+    {
+      "Name": "AskPrice3",
+      "Type": "double",
+      "Value": "AskPrices[2]",
+      "Equals": null
+    },
+    {
+      "Name": "AskQuantity3",
+      "Type": "double",
+      "Value": "AskQuantities[2]",
+      "Equals": null
+    },
+    {
+      "Name": "AskPrice4",
+      "Type": "double",
+      "Value": "AskPrices[3]",
+      "Equals": null
+    },
+    {
+      "Name": "AskQuantity4",
+      "Type": "double",
+      "Value": "AskQuantities[3]",
+      "Equals": null
+    },
+    {
+      "Name": "AskPrice5",
+      "Type": "double",
+      "Value": "AskPrices[4]",
+      "Equals": null
+    },
+    {
+      "Name": "AskQuantity5",
+      "Type": "double",
+      "Value": "AskQuantities[4]",
+      "Equals": null
+    }
+  ]
+}

src/DataStreamSuite/SharedStreams/RowTemplates/OrderImbalanceSchema.json

@@ -0,0 +1,200 @@
+{
+  "IsTimeSeries": true,
+  "Columns": [
+    {
+      "Name": "Id",
+      "Type": "int",
+      "Key": "id"
+    },
+    {
+      "Name": "Timestamp",
+      "Type": "DateTime"
+    },
+    {
+      "Name": "Venue",
+      "Type": "string"
+    },
+    {
+      "Name": "Symbol",
+      "Type": "string"
+    },
+    {
+      "Name": "BidPrices",
+      "Key": null,
+      "Type": "double[]",
+      "Value": "{ get; } = new double[5];",
+      "Equals": "BidPrices.SequenceEqual(other.BidPrices)"
+    },
+    {
+      "Name": "BidQuantities",
+      "Key": null,
+      "Type": "double[]",
+      "Value": "{ get; } = new double[5];",
+      "Equals": "BidQuantities.SequenceEqual(other.BidQuantities)"
+    },
+    {
+      "Name": "BidPrice1",
+      "Type": "double",
+      "Value": "BidPrices[0]",
+      "Equals": null
+    },
+    {
+      "Name": "BidQuantity1",
+      "Type": "double",
+      "Value": "BidQuantities[0]",
+      "Equals": null
+    },
+    {
+      "Name": "BidPrice2",
+      "Type": "double",
+      "Value": "BidPrices[1]",
+      "Equals": null
+    },
+    {
+      "Name": "BidQuantity2",
+      "Type": "double",
+      "Value": "BidQuantities[1]",
+      "Equals": null
+    },
+    {
+      "Name": "BidPrice3",
+      "Type": "double",
+      "Value": "BidPrices[2]",
+      "Equals": null
+    },
+    {
+      "Name": "BidQuantity3",
+      "Type": "double",
+      "Value": "BidQuantities[2]",
+      "Equals": null
+    },
+    {
+      "Name": "BidPrice4",
+      "Type": "double",
+      "Value": "BidPrices[3]",
+      "Equals": null
+    },
+    {
+      "Name": "BidQuantity4",
+      "Type": "double",
+      "Value": "BidQuantities[3]",
+      "Equals": null
+    },
+    {
+      "Name": "BidPrice5",
+      "Type": "double",
+      "Value": "BidPrices[4]",
+      "Equals": null
+    },
+    {
+      "Name": "BidQuantity5",
+      "Type": "double",
+      "Value": "BidQuantities[4]",
+      "Equals": null
+    },
+    {
+      "Name": "AskPrices",
+      "Key": null,
+      "Type": "double[]",
+      "Value": "{ get; } = new double[5];",
+      "Equals": "AskPrices.SequenceEqual(other.AskPrices)"
+    },
+    {
+      "Name": "AskQuantities",
+      "Key": null,
+      "Type": "double[]",
+      "Value": "{ get; } = new double[5];",
+      "Equals": "AskQuantities.SequenceEqual(other.AskQuantities)"
+    },
+    {
+      "Name": "AskPrice1",
+      "Type": "double",
+      "Value": "AskPrices[0]",
+      "Equals": null
+    },
+    {
+      "Name": "AskQuantity1",
+      "Type": "double",
+      "Value": "AskQuantities[0]",
+      "Equals": null
+    },
+    {
+      "Name": "AskPrice2",
+      "Type": "double",
+      "Value": "AskPrices[1]",
+      "Equals": null
+    },
+    {
+      "Name": "AskQuantity2",
+      "Type": "double",
+      "Value": "AskQuantities[1]",
+      "Equals": null
+    },
+    {
+      "Name": "AskPrice3",
+      "Type": "double",
+      "Value": "AskPrices[2]",
+      "Equals": null
+    },
+    {
+      "Name": "AskQuantity3",
+      "Type": "double",
+      "Value": "AskQuantities[2]",
+      "Equals": null
+    },
+    {
+      "Name": "AskPrice4",
+      "Type": "double",
+      "Value": "AskPrices[3]",
+      "Equals": null
+    },
+    {
+      "Name": "AskQuantity4",
+      "Type": "double",
+      "Value": "AskQuantities[3]",
+      "Equals": null
+    },
+    {
+      "Name": "AskPrice5",
+      "Type": "double",
+      "Value": "AskPrices[4]",
+      "Equals": null
+    },
+    {
+      "Name": "AskQuantity5",
+      "Type": "double",
+      "Value": "AskQuantities[4]",
+      "Equals": null
+    },
+    {
+      "Name": "Imbalance1",
+      "Type": "double",
+      "Value": "BidQuantity1 - AskQuantity1",
+      "Equals": null
+    },
+    {
+      "Name": "Imbalance2",
+      "Type": "double",
+      "Value": "Imbalance1 + BidQuantity2 - AskQuantity2",
+      "Equals": null
+    },
+    {
+      "Name": "Imbalance3",
+      "Type": "double",
+      "Value": "Imbalance2 + BidQuantity3 - AskQuantity3",
+      "Equals": null
+    },
+    {
+      "Name": "Imbalance4",
+      "Type": "double",
+      "Value": "Imbalance3 + BidQuantity4 - AskQuantity4",
+      "Equals": null
+    },
+    {
+      "Name": "Imbalance5",
+      "Type": "double",
+      "Value": "Imbalance4 + BidQuantity5 - AskQuantity5",
+      "Equals": null
+    }
+  ]
+}

src/DataStreamSuite/SharedStreams/RowTemplates/OrderLatencySchema.json

@@ -0,0 +1,48 @@
+{
+  "IsTimeSeries": true,
+  "Columns": [
+    {
+      "Name": "Id",
+      "Type": "int",
+      "Key": "id"
+    },
+    {
+      "Name": "SendTimestamp",
+      "Type": "DateTime"
+    },
+    {
+      "Name": "Timestamp",
+      "Type": "DateTime"
+    },
+    {
+      "Name": "Latency",
+      "Type": "TimeSpan",
+      "Value": "Timestamp - SendTimestamp",
+      "Equals": null
+    },
+    {
+      "Name": "Venue",
+      "Type": "string"
+    },
+    {
+      "Name": "Symbol",
+      "Type": "string"
+    },
+    {
+      "Name": "Price",
+      "Type": "double"
+    },
+    {
+      "Name": "Quantity",
+      "Type": "double"
+    },
+    {
+      "Name": "Flags",
+      "Type": "string"
+    },
+    {
+      "Name": "SequenceNumber",
+      "Type": "int"
+    }
+  ]
+}

src/DataStreamSuite/SharedStreams/RowTemplates/OrderLifespanSchema.json

@@ -0,0 +1,62 @@
+{
+  "IsTimeSeries": true,
+  "Columns": [
+    {
+      "Name": "Id",
+      "Type": "int",
+      "Key": "id"
+    },
+    {
+      "Name": "Venue",
+      "Type": "string"
+    },
+    {
+      "Name": "Symbol",
+      "Type": "string"
+    },
+    {
+      "Name": "Timestamp",
+      "Type": "DateTime"
+    },
+    {
+      "Name": "SendTimestamp",
+      "Type": "DateTime"
+    },
+    {
+      "Name": "Latency",
+      "Type": "TimeSpan",
+      "Value": "Timestamp - SendTimestamp",
+      "Equals": null
+    },
+    {
+      "Name": "FinalTimestamp",
+      "Type": "DateTime"
+    },
+    {
+      "Name": "Lifespan",
+      "Type": "TimeSpan",
+      "Value": "FinalTimestamp - Timestamp",
+      "Equals": null
+    },
+    {
+      "Name": "State",
+      "Type": "string"
+    },
+    {
+      "Name": "Side",
+      "Type": "bts.api.Side"
+    },
+    {
+      "Name": "Price",
+      "Type": "double"
+    },
+    {
+      "Name": "Quantity",
+      "Type": "double"
+    },
+    {
+      "Name": "FilledQuantity",
+      "Type": "double"
+    }
+  ]
+}

src/DataStreamSuite/SharedStreams/RowTemplates/RefTypeRow.tt

@@ -0,0 +1,121 @@
+<#@ template language="C#" #>
+<#@ output extension=".cs" #>
+<#@ parameter name="TableName" type="System.String" #>
+<#@ parameter name="TypeName" type="System.String" #>
+<#@ parameter name="SchemaJson" type="System.String" #>
+<#@ import namespace="System.Text.Json" #>
+<#@ import namespace="System.Text.Json.Nodes" #>
+<#
+   System.Console.WriteLine($"Reading json file '{SchemaJson}'");
+   var schemaStr = System.IO.File.ReadAllText(SchemaJson);
+   var schemaJ = JsonSerializer.Deserialize<JsonObject>(schemaStr);
+   if (schemaJ is null) {
+      throw new ArgumentException($"Failed to parse json from '{schemaJ}'");
+   }
+   if (!schemaJ.TryGetPropertyValue("Columns", out JsonNode? jn)) {
+      throw new ArgumentException($"Failed to get 'Columns' field from '{schemaJ}'");
+   }
+   var columnsJArray = jn.AsArray();
+   if (columnsJArray is null) {
+      throw new ArgumentException($"Failed to get 'Columns' field from '{schemaJ}'");
+   }
+   var columns = new System.Collections.Generic.List<System.Tuple<string, string, string, string, string>>();
+   for (int colIdx = 0; colIdx < columnsJArray.Count; colIdx++) {
+      var columnJ = columnsJArray[colIdx].AsObject();
+      string name = columnJ["Name"].GetValue<string>();
+      string type = columnJ["Type"].GetValue<string>();
+      string key = columnJ.TryGetPropertyValue("Key", out var keyJ) ? keyJ.GetValue<string>() : name;
+      string getter = columnJ.TryGetPropertyValue("Getter", out var getterJ) ? getterJ.GetValue<string>() : "{ get; init; }";
+      string equalsOp = columnJ.TryGetPropertyValue("Equals", out var equalsJ) ? equalsJ.GetValue<string>() : $"EqualityComparer<{type}>.Default.Equals({name}, other.{name})";
+      columns.Add(new(name, type, key, getter, equalsOp));
+   }
+   bool isTimeSeries = schemaJ.TryGetPropertyValue("IsTimeSeries", out jn) && jn.GetValue<bool>();
+#>
+#nullable disable
+namespace bts.SilverPine.Tables;
+
+using BCG;
+
+using System;
+
+public class <#=TypeName#> : <#=isTimeSeries ? "ITimeSeriesRow" : "IBlueRow" #>, IEquatable<<#=TypeName#>> {
+   public static IBlueTableSchema Schema { get; } =
+      new BlueTableSchema(
+         new ColumnInfo[] {
+      <# 
+      // Generate properties
+      for (int i = 0; i < columns.Count; i++) {
+         var type = columns[i].Item2;
+         var key = columns[i].Item3;
+      #>
+      new("<#=key#>", <#=i#>, typeof(<#=type#>)),
+      <#
+   }
+   #>
+   },
+         "<#= TableName #>");
+
+   IBlueTableSchema IBlueRow.schema => Schema;
+
+   <# 
+   // Generate properties
+   foreach (var column in columns) {
+      var name = column.Item1;
+      var type = column.Item2;
+      var getter = column.Item4;
+   #>
+public <#= type #> <#= name #> <#= getter #>
+   <# 
+   }
+   #>
+
+   // Indexer by column name
+   public object? this[string col] => col switch {
+      <# 
+      // Generate case statements for each column name
+      for (int i = 0; i < columns.Count; i++) {
+         var name = columns[i].Item1;
+         var key = columns[i].Item3;
+      #>
+"<#= key #>" => <#= name #>,
+      <# 
+} 
+      #>
+_ => throw new IndexOutOfRangeException()
+   };
+
+   // Indexer by column index
+   public object? this[int col] => col switch {
+      <# 
+      // Generate case statements for each column index
+      for (int i = 0; i < columns.Count; i++) {
+          var name = columns[i].Item1;
+      #>
+<#= i #> => <#= name #>,
+      <# 
+} 
+      #>
+_ => throw new KeyNotFoundException()
+   };
+
+   public bool Equals(<#= TypeName #> other) =>
+      other is not null
+   <# 
+   // Generate properties
+   for (int i = 0; i < columns.Count; i++) {
+      var equalsOp = columns[i].Item5;
+      if (string.IsNullOrEmpty(equalsOp)) {
+         continue;
+      }
+      equalsOp = equalsOp.Insert(0, "&& ");
+   #>
+   <#=equalsOp#>
+   <#}#>
+   ;
+
+   public override bool Equals(object? obj) =>
+      obj is <#=TypeName#> other && Equals(other);
+
+   public override int GetHashCode() =>
+      Id;
+}

src/DataStreamSuite/SharedStreams/RowTemplates/RefTypeVariableColumnRow.tt

@@ -0,0 +1,152 @@
+<#@ template language="C#" #>
+<#@ output extension=".cs" #>
+<#@ parameter name="TableName" type="System.String" #>
+<#@ parameter name="TypeName" type="System.String" #>
+<#@ parameter name="SchemaJson" type="System.String" #>
+<#@ import namespace="System" #>
+<#@ import namespace="System.Text.Json" #>
+<#@ import namespace="System.Text.Json.Nodes" #>
+<#
+   System.Console.WriteLine($"Reading json file '{SchemaJson}'");
+   var schemaStr = System.IO.File.ReadAllText(SchemaJson);
+   System.Console.WriteLine("Dingo");
+   var schemaJ = JsonSerializer.Deserialize<JsonObject>(schemaStr);
+   if (schemaJ is null) {
+      throw new ArgumentException($"Failed to parse json from '{schemaJ}'");
+   }
+   System.Console.WriteLine("Blingo");
+   if (!schemaJ.TryGetPropertyValue("Columns", out JsonNode? jn)) {
+      throw new ArgumentException($"Failed to get 'Columns' field from '{schemaJ}'");
+   }
+   var columnsJArray = jn.AsArray();
+   if (columnsJArray is null) {
+      throw new ArgumentException($"Failed to get 'Columns' field from '{schemaJ}'");
+   }
+   System.Console.WriteLine("Ringo");
+   var columns = new System.Collections.Generic.List<System.Tuple<string, string, string, string, string>>();
+   for (int colIdx = 0; colIdx < columnsJArray.Count; colIdx++) {
+      var columnJ = columnsJArray[colIdx].AsObject();
+      string name = columnJ["Name"].GetValue<string>();
+      System.Console.WriteLine($"Adding prop {name}");
+      string type = columnJ["Type"]?.GetValue<string>();
+      string key = columnJ.TryGetPropertyValue("Key", out var keyJ) ? keyJ?.GetValue<string>() : name;
+      string value = null;
+      if (columnJ.TryGetPropertyValue("Value", out var valueJ)) {
+         value = valueJ.GetValue<string>().Trim();
+         if (!value.StartsWith('{')) {
+            value = $"=> {valueJ.GetValue<string>()};";
+         }
+      } else {
+         value = "{ get; init; }";
+      }
+      string equalsOp = null;
+      if (columnJ.TryGetPropertyValue("Equals", out var equalsJ)) {
+         equalsOp = equalsJ?.GetValue<string>();
+      } else {
+         equalsOp = $"EqualityComparer<{type}>.Default.Equals({name}, other.{name})";
+      }
+      columns.Add(new(name, type, key, value, equalsOp));
+   }
+   bool isTimeSeries = schemaJ.TryGetPropertyValue("IsTimeSeries", out jn) && jn.GetValue<bool>();
+#>
+#nullable disable
+namespace bts.SilverPine.Tables;
+
+using BCG;
+
+using System;
+
+public class <#=TypeName#> : <#=isTimeSeries ? "ITimeSeriesRow" : "IBlueRow" #>, IEquatable<<#=TypeName#>> {
+   public static IBlueTableSchema Schema { get; } =
+      new BlueTableSchema(
+         new ColumnInfo[] {
+      <# 
+      // Generate properties
+      int offset = 0;
+      for (int i = 0; i < columns.Count; i++) {
+         var type = columns[i].Item2;
+         var key = columns[i].Item3;
+         if (key is null) {
+            offset++;
+            continue;
+         }
+      #>
+      new("<#=key#>", <#=i - offset#>, typeof(<#=type#>)),
+      <#
+   }
+   #>
+   },
+         "<#= TableName #>");
+   IBlueTableSchema IBlueRow.schema => Schema;
+
+   <# 
+   // Generate properties
+   foreach (var column in columns) {
+      var name = column.Item1;
+      var type = column.Item2;
+      var getter = column.Item4;
+   #>
+public <#= type #> <#= name #> <#= getter #>
+   <# 
+   }
+   #>
+
+   // Indexer by column name
+   public object? this[string col] => col switch {
+      <# 
+      // Generate case statements for each column name
+      for (int i = 0; i < columns.Count; i++) {
+         var name = columns[i].Item1;
+         var key = columns[i].Item3;
+         if (key is null) {
+            continue;
+         }
+      #>
+"<#= key #>" => <#= name #>,
+      <# 
+} 
+      #>
+_ => throw new IndexOutOfRangeException()
+   };
+
+   // Indexer by column index
+   public object? this[int col] => col switch {
+      <# 
+      // Generate case statements for each column index
+      offset = 0;
+      for (int i = 0; i < columns.Count; i++) {
+         var name = columns[i].Item1;
+         var key = columns[i].Item3;
+         if (key is null) {
+            offset++;
+            continue;
+         }
+      #>
+<#= i - offset #> => <#= name #>,
+      <# 
+} 
+      #>
+_ => throw new KeyNotFoundException()
+   };
+
+   public bool Equals(<#= TypeName #> other) =>
+      other is not null
+   <# 
+   // Generate properties
+   for (int i = 0; i < columns.Count; i++) {
+      var equalsOp = columns[i].Item5;
+      if (string.IsNullOrEmpty(equalsOp)) {
+         continue;
+      }
+      equalsOp = equalsOp.Insert(0, "&& ");
+   #>
+   <#=equalsOp#>
+   <#}#>
+   ;
+
+   public override bool Equals(object? obj) =>
+      obj is <#=TypeName#> other && Equals(other);
+
+   public override int GetHashCode() =>
+      Id;
+}

src/DataStreamSuite/SharedStreams/RowTemplates/ValueTypeRow.tt

@@ -0,0 +1,123 @@
+<#@ template language="C#" #>
+<#@ output extension=".cs" #>
+<#@ parameter name="TableName" type="System.String" #>
+<#@ parameter name="TypeName" type="System.String" #>
+<#@ parameter name="SchemaJson" type="System.String" #>
+<#@ import namespace="System.Text.Json" #>
+<#@ import namespace="System.Text.Json.Nodes" #>
+<#
+   System.Console.WriteLine($"Reading json file '{SchemaJson}'");
+   var schemaStr = System.IO.File.ReadAllText(SchemaJson);
+   var schemaJ = JsonSerializer.Deserialize<JsonObject>(schemaStr);
+   if (schemaJ is null) {
+      throw new ArgumentException($"Failed to parse json from '{schemaJ}'");
+   }
+   if (!schemaJ.TryGetPropertyValue("Columns", out JsonNode? columnsJNode)) {
+      throw new ArgumentException($"Failed to get 'Columns' field from '{schemaJ}'");
+   }
+   var columnsJArray = columnsJNode.AsArray();
+   if (columnsJArray is null) {
+      throw new ArgumentException($"Failed to get 'Columns' field from '{schemaJ}'");
+   }
+   var columns = new System.Collections.Generic.List<System.Tuple<string, string, string, string, string>>();
+   for (int colIdx = 0; colIdx < columnsJArray.Count; colIdx++) {
+      var columnJ = columnsJArray[colIdx].AsObject();
+      string name = columnJ["Name"].GetValue<string>();
+      string type = columnJ["Type"].GetValue<string>();
+      string key = columnJ.TryGetPropertyValue("Key", out var keyJ) ? keyJ.GetValue<string>() : name;
+      string getter = columnJ.TryGetPropertyValue("Getter", out var getterJ) ? getterJ.GetValue<string>() : "{ get; init; }";
+      string equalsOp = columnJ.TryGetPropertyValue("Equals", out var equalsJ) ? equalsJ.GetValue<string>() : $"EqualityComparer<{type}>.Default.Equals({name}, other.{name})";
+      columns.Add(new(name, type, key, getter, equalsOp));
+   }
+#>
+namespace bts.SilverPine.Tables;
+
+using BCG;
+
+using System;
+
+public readonly struct <#=TypeName#> : IBlueRow, IEquatable<<#=TypeName#>> {
+   public static IBlueTableSchema Schema { get; } =
+      new BlueTableSchema(
+         new ColumnInfo[] {
+      <# 
+      // Generate properties
+      for (int i = 0; i < columns.Count; i++) {
+         var type = columns[i].Item2;
+         var key = columns[i].Item3;
+      #>
+      new("<#=key#>", <#=i#>, typeof(<#=type#>)),
+      <#
+   }
+   #>
+   },
+         "<#= TableName #>");
+
+   IBlueTableSchema IBlueRow.schema => Schema;
+
+   <# 
+   // Generate properties
+   foreach (var column in columns) {
+      var name = column.Item1;
+      var type = column.Item2;
+      var getter = column.Item4;
+   #>
+public <#= type #> <#= name #> <#= getter #>
+   <# 
+   }
+   #>
+
+   // Indexer by column name
+   public readonly object? this[string col] => col switch {
+      <# 
+      // Generate case statements for each column name
+      for (int i = 0; i < columns.Count; i++) {
+         var name = columns[i].Item1;
+         var key = columns[i].Item3;
+      #>
+"<#= key #>" => <#= name #>,
+      <# 
+} 
+      #>
+_ => throw new IndexOutOfRangeException()
+   };
+
+   // Indexer by column index
+   public readonly object? this[int col] => col switch {
+      <# 
+      // Generate case statements for each column index
+      for (int i = 0; i < columns.Count; i++) {
+          var name = columns[i].Item1;
+      #>
+<#= i #> => <#= name #>,
+      <# 
+} 
+      #>
+_ => throw new KeyNotFoundException()
+   };
+
+   public bool Equals(<#= TypeName #> other) =>
+   <# 
+   // Generate properties
+   bool isFirstComp = true;
+   for (int i = 0; i < columns.Count; i++) {
+      var equalsOp = columns[i].Item5;
+      if (string.IsNullOrEmpty(equalsOp)) {
+         continue;
+      }
+      if (!isFirstComp) {
+         equalsOp = equalsOp.Insert(0, "&& ");
+      } else {
+         isFirstComp = false;
+      }
+   #>
+   <#=equalsOp#>
+   <#}#>
+   ;
+
+   public override bool Equals(object? obj) =>
+      obj is <#=TypeName#> other && Equals(other);
+
+   public override int GetHashCode() =>
+      Id;
+}

src/DataStreamSuite/SharedStreams/Services/NaturalLanguageStreamService.cs

@@ -0,0 +1,43 @@
+// namespace Bts.SharedStreams.Services;
+
+// using BCG;
+// using BCG.Utilities.General;
+
+// using bts.SilverPine;
+
+
+// using Microsoft.Extensions.Hosting;
+// using Microsoft.Extensions.Options;
+
+// using System;
+// using System.Collections.Generic;
+// using System.Threading;
+// using System.Threading.Tasks;
+
+// /// <summary>
+// /// An <see cref="IHostedService"/> that imports a collection of <see cref="IWorker"/>s and
+// /// calls each <see cref="IWorker.doWork"/> in a loop.
+// /// </summary>
+// public class NaturalLanguageStreamService : BackgroundService {
+//    readonly IWorker masterWorker;
+//    readonly NLPromptConfig config;
+
+//    public NaturalLanguageStreamService(IOptions<NLPromptConfig> config)
+//       : this(config.Value) { }
+
+//    public NaturalLanguageStreamService(NLPromptConfig config) {
+//       this.config = config;
+//    }
+
+//    protected override Task ExecuteAsync(CancellationToken cancellationToken) {
+//       return Task.Factory.StartNew(
+//          Execute, cancellationToken, TaskCreationOptions.LongRunning, TaskScheduler.Default);
+
+//       void Execute() {
+//          while (!cancellationToken.IsCancellationRequested) {
+//             masterWorker.doWork();
+//             Thread.Sleep(config.Interval);
+//          }
+//       }
+//    }
+// }

src/DataStreamSuite/SharedStreams/Services/SharedStreamsServices.cs

@@ -0,0 +1,36 @@
+namespace Bts.SharedStreams.Services;
+
+using bts.SilverPine;
+
+using Bts.DataStream;
+
+using Microsoft.Extensions.Configuration;
+using Microsoft.Extensions.DependencyInjection;
+
+public static class SharedStreamsServices {
+   public static IServiceCollection AddOhlcvStreams(this IServiceCollection services, IConfigurationSection configurationSection) =>
+      services.Configure<DatabaseConfig>(configurationSection)
+         .addStreamFactory<OhlcvStreamFactory>()
+         .addStreamFactory<BboStreamFactory>()
+         .addStreamFactory<SettlementStreamFactory>()
+         .addStreamFactory<MbpStreamFactory>()
+         .addStreamFactory<MboStreamFactory>()
+         .addStreamFactory<OrderLatencyStreamFactory>()
+         .addStreamFactory<AvgOrderLatencyStreamFactory>()
+         .addStreamFactory<OrderImbalanceStreamFactory>()
+         .addStreamFactory<OrderLifespanStreamFactory>()
+         ;
+
+   public static IServiceCollection AddNaturalLanguageStreams(this IServiceCollection services, IConfigurationSection configurationSection) {
+      services.Configure<NLPromptConfig>(configurationSection);
+      NLPromptConfig.SetGlobalConfig(configurationSection.Get<NLPromptConfig>() ?? NLPromptConfig.Default());
+      // TODO: integrate config into a NLStream provider (so it can provide config values
+      // instead of the ViewStream's getting them from a static singleton)
+      return services.AddSingleton<NLStreamFactory>()
+         .AddSingleton<IDataStreamFactory, NLStreamFactory>(sp => sp.GetRequiredService<NLStreamFactory>());
+   }
+
+   static IServiceCollection addStreamFactory<T>(this IServiceCollection services) where T : class, IDataStreamFactory =>
+      services.AddSingleton<T>()
+         .AddSingleton<IDataStreamFactory, T>(sp => sp.GetRequiredService<T>());
+}

src/DataStreamSuite/SharedStreams/SharedStreams.csproj

@@ -16,8 +16,19 @@
   </ItemGroup>
 
   <ItemGroup>
-  <PackageReference Include="MathNet.Numerics" Version="5.0.0" />
-  <PackageReference Include="OpenAI" Version="2.0.0" />
+    <PackageReference Include="MathNet.Numerics" Version="5.0.0" />
+    <PackageReference Include="OpenAI" Version="2.0.0" />
+    <PackageReference Include="System.Linq.Async" Version="6.0.1" />
+  </ItemGroup>
+
+  <ItemGroup>
+    <RowType Include="RowTemplates\RefTypeRow.tt" SchemaJson="RowTemplates\BboSchema.json" TypeName="BboRow" TableName="Bbo" />
+    <RowType Include="RowTemplates\RefTypeVariableColumnRow.tt" SchemaJson="RowTemplates\MbpSchema.json" TypeName="MbpRow" TableName="Mbp" />
+    <RowType Include="RowTemplates\RefTypeRow.tt" SchemaJson="RowTemplates\MboSchema.json" TypeName="MboRow" TableName="Mbo" />
+    <RowType Include="RowTemplates\RefTypeVariableColumnRow.tt" SchemaJson="RowTemplates\OrderImbalanceSchema.json" TypeName="OrderImbalanceRow" TableName="OrderImbalance" />
+    <RowType Include="RowTemplates\RefTypeVariableColumnRow.tt" SchemaJson="RowTemplates\OrderLatencySchema.json" TypeName="OrderLatencyRow" TableName="OrderLatency" />
+    <RowType Include="RowTemplates\RefTypeVariableColumnRow.tt" SchemaJson="RowTemplates\AvgOrderLatencySchema.json" TypeName="AvgOrderLatencyRow" TableName="AvgOrderLatency" />
+    <RowType Include="RowTemplates\RefTypeVariableColumnRow.tt" SchemaJson="RowTemplates\OrderLifespanSchema.json" TypeName="OrderLifespanRow" TableName="OrderLifespan" />
   </ItemGroup>
 
 </Project>

src/DataStreamSuite/SharedStreams/StreamFactories/AvgOrderLatencyStreamFactory.cs

@@ -0,0 +1,106 @@
+namespace bts.SilverPine;
+
+using BCG;
+
+using bts.api.core;
+using bts.SilverPine.StreamFactory.Logging;
+using bts.SilverPine.Tables;
+
+using Bts.DataStream;
+
+using Microsoft.Extensions.Logging;
+using Microsoft.Extensions.Options;
+
+using System;
+using System.Reactive.Linq;
+
+public class AvgOrderLatencyStreamFactory : IDataStreamFactory {
+   readonly DatabaseConfig config;
+   readonly ILogger logger;
+
+   public AvgOrderLatencyStreamFactory(IOptions<DatabaseConfig> config, ILogger<AvgOrderLatencyStreamFactory> logger)
+      : this(config.Value, logger) { }
+
+   public AvgOrderLatencyStreamFactory(DatabaseConfig config, ILogger<AvgOrderLatencyStreamFactory> logger) {
+      this.config = config;
+      this.logger = logger;
+   }
+
+   public static IObservable<TableUpdate<AvgOrderLatencyRow, int>>? CreateAvgOrderLatencyTables(
+      DatabaseConfig config, HistoricDataParams args, ILogger logger) {
+      args.Validate().ValidateInterval();
+      var orderLatencyParams = new OrderLatencyStreamParams() {
+         Venue = args.Venue,
+         Symbol = args.Symbol,
+         BeginDate = args.BeginDate,
+         EndDate = args.EndDate,
+      };
+      return OrderLatencyStreamFactory.CreateOrderLatencyTables(config, orderLatencyParams, logger)
+         ?.BufferTable(TimeSpan.FromMilliseconds(500))
+         ?.GroupTable(
+            r => r.Id,
+            r => (int)((r.Timestamp.Ticks - args.BeginDate.Ticks) / args.Interval!.Value.Ticks))
+         ?.SelectKeyValue(ComputeAvgLatency, r => r.Id);
+
+      AvgOrderLatencyRow ComputeAvgLatency(TableUpdateGrouping<OrderLatencyRow, int, int> group) {
+         double avgLatency = 0;
+         double latencySum = 0;
+         int nTotal = 0;
+         int n = 0;
+         foreach (var r in group.table.state.Values) {
+            if ((double.MaxValue - latencySum) > r.Latency.Ticks) {
+               latencySum += r.Latency.Ticks;
+               n++;
+            } else if (n > 0) {
+               nTotal += n;
+               avgLatency += ((latencySum / n) - avgLatency) * (n / nTotal);
+               latencySum = 0;
+               n = 0;
+            } else {
+               throw new InvalidOperationException("Latency is too long (unable to compute the avg)");
+            }
+         }
+         if (n > 0) {
+            nTotal += n;
+            avgLatency += ((latencySum / n) - avgLatency) * (n / nTotal);
+            latencySum = 0;
+            n = 0;
+         }
+         return new AvgOrderLatencyRow() {
+            Symbol = args.Symbol,
+            Venue = args.Venue,
+            Id = group.group_key,
+            AvgLatency = new((long)avgLatency),
+            Timestamp = args.BeginDate + new TimeSpan(group.group_key * args.Interval!.Value.Ticks),
+            OrderCount = group.table.state.Count,
+            Volume = (long)group.table.state.Values.Sum(r => r.Quantity)
+         };
+      }
+   }
+
+   public ViewStream? Create(string streamId, IEnumerable<(string? Name, int? Index, object? Value)> args) {
+      if (SupportedStreams.AvgOrderLatency.IsMatch(streamId)) {
+         return createOrderLatency(args);
+      }
+      return null;
+   }
+
+   public IEnumerable<IDataStreamMetadata> GetSupportedStreams() {
+      yield return SupportedStreams.AvgOrderLatency;
+   }
+
+   ViewStream? createOrderLatency(IEnumerable<(string? Name, int? Index, object? Value)> args) {
+      if (!StreamFactoryArgStore.TryGetParams(args, out OrderLatencyStreamParams? sp, out var err)) {
+         logger.LogInvalidParams("avgOrderLatency", err);
+         return null;
+      }
+      return OrderLatencyStreamFactory.CreateOrderLatencyTables(config, sp, logger)
+         ?.AsViewStream(AvgOrderLatencyRow.Schema);
+   }
+
+   static class SupportedStreams {
+      public static readonly DataStreamMetadata AvgOrderLatency = new() {
+         Name = AvgOrderLatencyRow.Schema.tableName
+      };
+   }
+}

src/DataStreamSuite/SharedStreams/StreamFactories/AvgOrderLatencyStreamParams.cs

@@ -0,0 +1,34 @@
+namespace bts.SilverPine;
+
+using System.Diagnostics.CodeAnalysis;
+
+public class AvgOrderLatencyStreamParams {
+   [StreamArgument("venue", "exchange", "exch", IsRequired = true)]
+   public string? Venue { get; set; }
+
+   [StreamArgument("symbol", "sym", "product", "asset", IsRequired = true)]
+   public string? Symbol { get; set; }
+
+   [StreamArgument("begin", "beginDate", "start", "startDate", "from", IsRequired = true)]
+   public DateTime BeginDate { get; set; }
+
+   [StreamArgument("end", "endDate", "stop", "stopDate", "to", IsRequired = true)]
+   public DateTime EndDate { get; set; }
+
+   [StreamArgument("interval", IsRequired = false)]
+   public TimeSpan Interval { get; set; } = TimeSpan.Zero;
+
+   [MemberNotNull(nameof(Venue), nameof(Symbol))]
+   public AvgOrderLatencyStreamParams Validate() {
+      if (Venue is null) {
+         throw new InvalidOperationException("Venue is null");
+      }
+      if (Symbol is null) {
+         throw new InvalidOperationException("Symbol is null");
+      }
+      if (Interval < TimeSpan.Zero) {
+         throw new InvalidOperationException("Interval < TimeSpan.Zero");
+      }
+      return this;
+   }
+}

src/DataStreamSuite/SharedStreams/StreamFactories/BboStreamFactory.cs

@@ -0,0 +1,80 @@
+namespace bts.SilverPine;
+
+using BCG;
+
+using bts.api.core;
+using bts.SilverPine.StreamFactory.Logging;
+using bts.SilverPine.Tables;
+
+using Bts.DataStream;
+
+using Microsoft.Extensions.Logging;
+using Microsoft.Extensions.Options;
+
+using System;
+using System.Collections.Generic;
+
+public class BboStreamFactory : IDataStreamFactory {
+   readonly DatabaseConfig config;
+   readonly ILogger logger;
+
+   public BboStreamFactory(IOptions<DatabaseConfig> config, ILogger<BboStreamFactory> logger)
+      : this(config.Value, logger) { }
+
+   public BboStreamFactory(DatabaseConfig config, ILogger<BboStreamFactory> logger) {
+      this.config = config;
+      this.logger = logger;
+   }
+
+   public static IObservable<TableUpdate<BboRow, int>>? CreateBboTables(DatabaseConfig config, BboStreamParams args, ILogger logger) {
+      args.Validate();
+      return new HistoricDataLoader(config, logger).LoadTablesObservable(
+         new HistoricDataParams() {
+            DataType = args.DataType,
+            Venue = args.Venue,
+            Symbol = args.Symbol,
+            BeginDate = args.BeginDate,
+            EndDate = args.EndDate,
+            Interval = args.Interval,
+            TsNanosCol = "ts_event_ns"
+         },
+         RowFactory);
+
+      BboRow RowFactory(IBlueRow row, int groupKey, DateTime timestamp) =>
+         new() {
+            Id = groupKey,
+            Timestamp = timestamp,
+            Symbol = args.Symbol,
+            Venue = args.Venue,
+            BidPrice = row.GetFloat64("bid_px_00") ?? double.NaN,
+            BidQuantity = row.GetFloat64("bid_sz_00") ?? double.NaN,
+            AskPrice = row.GetFloat64("ask_px_00") ?? double.NaN,
+            AskQuantity = row.GetFloat64("ask_sz_00") ?? double.NaN,
+         };
+   }
+
+   public ViewStream? Create(string streamId, IEnumerable<(string? Name, int? Index, object? Value)> args) {
+      if (SupportedStreams.Bbo.IsMatch(streamId)) {
+         return createBbo(args);
+      }
+      return null;
+   }
+
+   public IEnumerable<IDataStreamMetadata> GetSupportedStreams() {
+      yield return SupportedStreams.Bbo;
+   }
+
+   ViewStream? createBbo(IEnumerable<(string? Name, int? Index, object? Value)> args) {
+      if (!StreamFactoryArgStore.TryGetParams(args, out BboStreamParams? bsp, out var err)) {
+         logger.LogInvalidParams("bbo", err);
+         return null;
+      }
+      return CreateBboTables(config, bsp, logger)?.AsViewStream(BboRow.Schema);
+   }
+
+   static class SupportedStreams {
+      public static readonly DataStreamMetadata Bbo = new() {
+         Name = "bbo"
+      };
+   }
+}

src/DataStreamSuite/SharedStreams/StreamFactories/BboStreamParams.cs

@@ -0,0 +1,56 @@
+namespace bts.SilverPine;
+
+using System.Diagnostics.CodeAnalysis;
+
+public class BboStreamParams {
+   [StreamArgument("venue", "exchange", "exch", IsRequired = true)]
+   public string? Venue { get; set; }
+
+   [StreamArgument("symbol", "sym", "product", "asset", IsRequired = true)]
+   public string? Symbol { get; set; }
+
+   [StreamArgument("begin", "beginDate", "start", "startDate", "from", IsRequired = true)]
+   public DateTime BeginDate { get; set; }
+
+   [StreamArgument("end", "endDate", "stop", "stopDate", "to", IsRequired = true)]
+   public DateTime EndDate { get; set; }
+
+   [StreamArgument("interval", IsRequired = true)]
+   public TimeSpan Interval { get; set; }
+
+   // Normally, the dataType is inferred from Interval, though it can be explicitly specified as well.
+   public string? DataType { get; set; }
+
+   [MemberNotNull(nameof(Venue), nameof(Symbol), nameof(DataType))]
+   public BboStreamParams Validate() {
+      if (Venue is null) {
+         throw new InvalidOperationException("Venue is null");
+      }
+      if (Symbol is null) {
+         throw new InvalidOperationException("Symbol is null");
+      }
+      if (Interval <= TimeSpan.Zero) {
+         throw new InvalidOperationException("Interval <= TimeSpan.Zero");
+      }
+      DataType ??= DataTypeFromInterval(Interval);
+      if (DataType is null) {
+         throw new InvalidOperationException(
+            $"DataType is null. Interval {Interval} did not map to well-known data type (bbo1s, bbo1m, bbo1h, bbo1d).");
+      }
+      return this;
+   }
+
+   public static string? DataTypeFromInterval(TimeSpan interval) {
+      if (interval == TimeSpan.FromSeconds(1)) {
+         return "bbo1s";
+      } else if (interval == TimeSpan.FromMinutes(1)) {
+         return "bbo1m";
+      } else if (interval == TimeSpan.FromHours(1)) {
+         return "bbo1h";
+      } else if (interval == TimeSpan.FromDays(1)) {
+         return "bbo1d";
+      } else {
+         return null;
+      }
+   }
+}

src/DataStreamSuite/SharedStreams/StreamFactories/HistoricDataLoader.cs

@@ -0,0 +1,133 @@
+namespace bts.SilverPine;
+
+using BCG;
+
+using bts.api.core;
+
+using Bts.DataStream;
+using Bts.DataStreamCore;
+
+using Microsoft.Extensions.Logging;
+
+using System.Data;
+using System.Data.SQLite;
+using System.Linq;
+using System.Reactive.Linq;
+using System.Reflection;
+
+public class HistoricDataLoader {
+   readonly DatabaseConfig dbConfig;
+   readonly ILogger? logger;
+
+   public HistoricDataLoader(DatabaseConfig dbConfig, ILogger? logger) {
+      this.dbConfig = dbConfig;
+      this.logger = logger;
+   }
+
+   public IEnumerable<StreamQueryMetadata> LoadQueryMetadata() {
+      try {
+         var connectionStr = $"Data Source={dbConfig.Path};Version=3";
+         var dataTable = new DataTable();
+         using var conn = new SQLiteConnection($"Data Source={dbConfig.Path};Version=3");
+         conn.Open();
+         using var sqlCmd = new SQLiteCommand(dbConfig.Query, conn);
+         using var adapter = new SQLiteDataAdapter(sqlCmd);
+         adapter.Fill(dataTable);
+
+         return dataTable.Rows.OfType<DataRow>()
+            .Select(r => new StreamQueryMetadata() {
+               Symbol = r["symbol"] as string,
+               DataType = r["data_type"] as string,
+               BeginDate = DateTime.Parse(r["begin_date"]?.ToString()!),
+               EndDate = DateTime.Parse(r["end_date"]?.ToString()!),
+               Location = r["location"] as string,
+               Query = r["query"] as string,
+            });
+      } catch (Exception ex) {
+         logger?.LogError("Failed to load query metadata from '{dbPath}'. Query: {query}. Exception {ex}", dbConfig.Path, dbConfig.Query, ex);
+      }
+      return Enumerable.Empty<StreamQueryMetadata>();
+   }
+
+   public IAsyncEnumerable<IBlueRowCollection> LoadRowsAsync(HistoricDataParams args) {
+      args.Validate();
+      var venue = resolveVenue(args.Venue);
+      var queryMetadatas = LoadQueryMetadata().ToList();
+      var perDateQueries = queryMetadatas
+         .Where(m => m.DataType == args.DataType && m.Symbol == args.Symbol && m.BeginDate < args.EndDate && m.EndDate > args.BeginDate)
+         .ToList();
+      if (perDateQueries.Count == 0) {
+         return AsyncEnumerable.Empty<IBlueRowCollection>();
+      }
+      return perDateQueries.ToAsyncEnumerable().SelectMany(q => {
+         var location = q.Location ?? throw new InvalidOperationException($"'location' is null in metadata for {args.DataType} stream");
+         if (!Path.IsPathRooted(location)) {
+            var dir = Path.GetDirectoryName(dbConfig.Path) ?? Assembly.GetExecutingAssembly().Location;
+            location = Path.Combine(dir, location);
+         }
+         var query = q.Query ?? throw new InvalidOperationException($"'query' is null in metadata for {args.DataType} stream");
+         var andFilters = new List<string>(2);
+         if (args.BeginDate > DateTime.MinValue) {
+            var unixTicks = args.BeginDate.Ticks - DateTime.UnixEpoch.Ticks;
+            andFilters.Add($"{args.TsNanosCol} > {unixTicks * 100 - 1}");
+         }
+         if (args.EndDate < DateTime.MaxValue) {
+            var unixTicks = args.EndDate.Ticks - DateTime.UnixEpoch.Ticks;
+            andFilters.Add($"{args.TsNanosCol} < {unixTicks * 100}");
+         }
+         if (andFilters.Count > 0) {
+            if (query.EndsWith(';')) {
+               query = query.Substring(0, query.Length - 1);
+            }
+            query += " where " + string.Join(" and ", andFilters) + ';';
+         }
+         return new SQLiteEnumerable(location, query);
+      });
+   }
+
+   public IObservable<TableUpdate<T, int>>? LoadTablesObservable<T>(HistoricDataParams args, Func<IBlueRow, int, DateTime, T> rowFactory) where T : ITimeSeriesRow {
+      args.Validate();
+      var seed = TableUpdate<T, int>.mk_empty(
+         args.Interval.HasValue
+            ? r => GroupKey(r.Timestamp)
+            : r => r.Id);
+      return LoadRowsAsync(args).ToObservable().Scan(seed, AddPage);
+
+      int GroupKey(DateTime ts) =>
+         (int)((ts.Ticks - args.BeginDate.Ticks) / args.Interval!.Value.Ticks);
+
+      TableUpdate<T, int> AddPage(TableUpdate<T, int> res, IBlueRowCollection blueRows) {
+         var tsNanosIdx = blueRows.schema.ResolveIndices(args.TsNanosCol);
+         var bd = res.to_builder();
+         foreach (IBlueRow blueRow in blueRows) {
+            var tsNanos = blueRow.GetInt64(tsNanosIdx);
+            if (!tsNanos.HasValue) {
+               continue;
+            }
+            var ts = DateTime.UnixEpoch.Add(new TimeSpan(tsNanos.Value / 100));
+            if (ts < args.BeginDate || ts >= args.EndDate) {
+               continue;
+            }
+            if (!args.Interval.HasValue) {
+               bd.add_or_update(rowFactory(blueRow, blueRow.Id(), ts));
+               continue;
+            }
+            var groupKey = GroupKey(ts);
+            if (!bd.TryGetValue(groupKey, out var row) || row.Timestamp < ts) {
+               row = rowFactory(blueRow, groupKey, ts);
+               if (row is null) {
+                  continue;
+               }
+               bd.add_or_update(row);
+            }
+         }
+         return bd.to_table();
+      }
+   }
+
+   static string resolveVenue(string venue) =>
+      venue switch {
+         "CME" or "XCME" or "CBOT" or "XCBT" => "GLBX",
+         _ => venue
+      };
+}