diff --git a/KeyCode.conf b/KeyCode.conf
new file mode 100644
index 000000000..5ac45f897
--- /dev/null
+++ b/KeyCode.conf
@@ -0,0 +1,15 @@
+#nusers:
+#allow concurrent : true
+#expires: 2024-Oct-06 00:00:00 638637696000000000
+#generation_time: 2023-Oct-06 18:58:48.015723 5250008173707545134
+EncryptedKeyCode {
+ value {
+ keycode: "KgoMeiVPzf/kv65GYGCyQQJVY/11sId6LYRwNNlNwA83+sISiSQnOZrziSgxjEAcaAZQn1nV4hY6MRi4XXznvxLYnDKf2bK0yktWWpSSqxqNxbDQ6AVSk5MGHptbsQSGYpYHlyCi0Niv8ES5NTj/FiBJXPrGEZABZBbNkqXIOa0HDdcxY+joeIf32ekX+5/LXN5XDTZBFT2A7q+IPDrFbKzSBvlzGOONSlq+XyKZZPqt0rXGO45V/GxFUTMYiEvn1/72iPGQ715tFXkr9ozGtitonnixzvW8Sfwmu7vM8VpWfoAc9D/1w5m5Ol5BS+Je8wigQysyrC8rud2KU+f7bQ=="
+ }
+}
+
+PublicKey {
+ value {
+ key: "-----BEGIN PUBLIC KEY-----\nMIIBIjANBgkqhkiG9w0BAQEFAAOCAQ8AMIIBCgKCAQEAvvtV/IP1l/F85ME8N8H+\n8TCBOB68Lxd7sdR0geJGssLBq5F22IIlVKylNxiQV3TE6/hmL9dmr8FuUMrjPCal\nvma0axyAdyCMjFH3U0bYzzxnxSWuj5VJIgaZKOVQuX/3Hb+l4RZi7VWZgQFvEJ+F\ny66S2wDu4ey71Mt0yxkMsl5WMZBL3M1z+bt3pKzmBoyIMVLDcV1k1Yr2wspIRriZ\n95er3JWofK1P9s8B3OoqypEopLRW5hJaZN8YRpFNq5VwRUSj83KVobBAQTWDdOA5\nzOkixPVklZFaNyR06iT0gcgZUUM2bjyfQJGQ4GBCZpZb8KPPgtP+WhghoWby9OLv\nKQIDAQAB\n-----END PUBLIC KEY-----"
+ }
+}
diff --git a/cfgl b/cfgl
new file mode 100644
index 000000000..81dff3389
--- /dev/null
+++ b/cfgl
@@ -0,0 +1,33 @@
+diff.astextplain.textconv=astextplain
+filter.lfs.clean=git-lfs clean -- %f
+filter.lfs.smudge=git-lfs smudge -- %f
+filter.lfs.process=git-lfs filter-process
+filter.lfs.required=true
+http.sslbackend=openssl
+http.sslcainfo=C:/Program Files/Git/mingw64/etc/ssl/certs/ca-bundle.crt
+core.autocrlf=true
+core.fscache=true
+core.symlinks=false
+pull.rebase=false
+credential.helper=manager
+credential.https://dev.azure.com.usehttppath=true
+init.defaultbranch=master
+status.short=true
+core.repositoryformatversion=0
+core.filemode=false
+core.bare=false
+core.logallrefupdates=true
+core.symlinks=false
+core.ignorecase=true
+submodule.active=.
+remote.origin.url=git@bitbucket.org:bluetradesys/spark.git
+remote.origin.fetch=+refs/heads/*:refs/remotes/origin/*
+branch.spark_crypto.dev.remote=origin
+branch.spark_crypto.dev.merge=refs/heads/spark_crypto.dev
+submodule.docker/linux.url=git@bitbucket.org:bluetradesys/linux_docker.git
+submodule.docker/windows.url=git@bitbucket.org:bluetradesys/windows_docker.git
+submodule.externals.url=git@bitbucket.org:bluetradesys/externals.git
+branch.development.remote=origin
+branch.development.merge=refs/heads/development
+user.name=John Colvin
+user.email=john.colvin@bluetradesys.com
diff --git a/get-token.py b/get-token.py
new file mode 100644
index 000000000..ee22fcaaa
--- /dev/null
+++ b/get-token.py
@@ -0,0 +1,45 @@
+# import json
+# from rauth import OAuth2Service
+
+
+# class ExampleOAuth2Client:
+# def __init__(self, url, path, client_id, client_secret):
+# self.access_token = None
+# self.service = OAuth2Service(
+# name="foo",
+# client_id=client_id,
+# client_secret=client_secret,
+# access_token_url=f'{url}{path}',
+# authorize_url=f'{url}{path}',
+# base_url=url,
+# )
+# self.get_access_token()
+
+# def get_access_token(self):
+# data = {
+# 'grant_type': 'client_credentials', # generally required!
+# }
+# session = self.service.get_auth_session(data=data, decoder=json.loads)
+# self.access_token = session.access_token
+
+
+# def _main():
+# client_id = 'API_THINKALPHA123'
+# client_secret = '$MM9Rn*#qYARY$$QTMA#AMn*'
+# oath = ExampleOAuth2Client(
+# url='https://authnr.cmegroup.com',
+# path='/as/token.oath2',
+# client_id=client_id,
+# client_secret=client_secret,
+# )
+# tok = oath.get_access_token()
+# print(tok)
+
+
+# if __name__ == '__main__':
+# try:
+# _main()
+# except json.JSONDecodeError as jde:
+# print(f'get-token.py failed with json-decode-error: {jde}.\n{jde.doc}\n\nlineno: {jde.lineno}\npos: {jde.pos}')
+# except Exception as e:
+# print(f'get-token.py failed: {e}')
diff --git a/get-token2.py b/get-token2.py
new file mode 100644
index 000000000..64813dce2
--- /dev/null
+++ b/get-token2.py
@@ -0,0 +1,30 @@
+import subprocess
+
+
+def _main():
+ host = "markets.api.uat.cmegroup.com"
+ port = "443"
+ # AUTH_ENDPOINT="$host/marketdatastream/v1"
+ AUTH_ENDPOINT = 'https://authnr.cmegroup.com/as/token.oauth2'
+ API_KEY = "API_THINKALPHA123"
+ API_SECRET = '$MM9Rn*#qYARY$$QTMA#AMn*'
+
+ # Define your API key and secret
+ # API_KEY="YOUR_API_KEY"
+ # API_SECRET="YOUR_API_SECRET"
+
+ # Define the authentication API endpoint URL
+ # AUTH_ENDPOINT="https://api.example.com/auth"
+
+ # Perform the POST request and capture the response
+ # response=$(curl -s -X POST -d "api_key=$API_KEY&api_secret=$API_SECRET&grant_type=password&username=API_THINKALPHA123&password=$API_SECRET" "$AUTH_ENDPOINT")
+ # subprocess.run('ls')
+ subprocess.run(
+ f'curl -s -X POST -d "grant_type=client_credentials&client_id={API_KEY}&client_secret={API_SECRET}" "{AUTH_ENDPOINT}"',
+ )
+
+
+# curl --user 'API_THINKALPHA123:$MM9Rn*#qYARY$$QTMA#AMn*' 'https://query-api.datamine.cmegroup.com/v3/query/query_order_record?tradedate=2023-07-10&fields=symbol,transacttime,instrumentsequencenumber,tradeprice,tradequantity,tradeflag&q=asset.eq.TN,securitytype.eq.FUT,tradeflag.nin.OBS&fileFormat=csv&securityexchange=XCBT&underlyingproduct=Interest%20Rate'
+
+if __name__ == '__main__':
+ _main()
diff --git a/get-token3.py b/get-token3.py
new file mode 100644
index 000000000..e69de29bb
diff --git a/good-dev-readme.md b/good-dev-readme.md
new file mode 100644
index 000000000..a3c506609
--- /dev/null
+++ b/good-dev-readme.md
@@ -0,0 +1,211 @@
+[cmake]: https://cmake.org/
+[cmake ext]: https://marketplace.visualstudio.com/items?itemName=ms-vscode.cmake-tools
+[cmake presets]: https://cmake.org/cmake/help/latest/manual/cmake-presets.7.html
+[clangd ext]: https://marketplace.visualstudio.com/items?itemName=llvm-vs-code-extensions.vscode-clangd
+[cpp ext]: https://marketplace.visualstudio.com/items?itemName=ms-vscode.cpptools
+[docker]: https://www.docker.com/
+[dotnet]: https://dotnet.microsoft.com/en-us/download/dotnet
+[ninja build]: https://ninja-build.org/
+[ninja generator]: https://cmake.org/cmake/help/latest/generator/Ninja.html
+[ninja multi-config generator]: https://cmake.org/cmake/help/latest/generator/Ninja%20Multi-Config.html
+[python]: https://www.python.org/
+[py venv]: https://docs.python.org/3/library/venv.html
+[visual studio]: https://visualstudio.microsoft.com/
+[visual studio generator]: https://cmake.org/cmake/help/latest/generator/Visual%20Studio%2017%202022.html
+[devcontainer]: https://code.visualstudio.com/docs/devcontainers/create-dev-container
+
+# C++ Intellisense for Spark on Windows (VS Code)
+**Steps**
+1. Install the **[C++ extension][cpp ext]**
+2. Install the **[CMake extension][cmake ext]**
+3. Copy `c_cpp_properties.json` from this folder to `$workspaceFolder/.vscode/`
+4. In the VS Code Command Palette, run `CMake: Select Configure Preset`, then
+ select '**spark**' from the dropdown
+
+**Notes**
+* Intellisense won't work for generated files/types until you build libspark
+
+# C++ Intellisense for Sparkd on Linux (VS Code)
+1. Install the **[CMake extension][cmake ext]**
+2. Install the **[Clangd extension][clangd ext]**
+3. (Optional) Install the **[C++ extension][cpp ext]** (*needed for debugging,
+ but intellisense should work without it*).
+4. If the **[C++ extension][cpp ext]** is installed, disable its intellisense in
+ your workspace settings (*Clangd intellisense is better*)
+ ```
+ "C_Cpp.intelliSenseEngine": "disabled"
+ ```
+5. In the VS Code Command Palette, run **CMake: Select Configure Preset**, then select the
+ **[preset][cmake presets]** you wish to build from the dropdown
+ * *The **Clang** presets may work best with the **Clangd** extension*
+ * *Intellisense won't work for generated files/types until you build libspark*
+
+# Build Spark on Windows
+### **Dependencies**
+* **[CMake][cmake]** (version >= 3.23)
+* **[Python][python]** (version >= 3.9)
+* **[Visual Studio][visual studio]** (2022)
+* **[Dotnet][dotnet]** (sdk version >= 7.0.0)
+
+### **References**
+* **[CMakePresets][cmake presets]**
+* **[Visual Studio 17 2022 (CMake generator)][visual studio generator]**
+* **[Python Virtual Environment][py venv]**
+> *Run commands from the repo root dir unless otherwise noted*
+1. Set up the **[Python virtual environment][py venv]**
+ ```
+ # Create the virtual environment
+ python3 -m venv .venv
+
+ # Activate the virtual environment
+ .venv\bin\activate.bat
+
+ # Update pip
+ python -m pip install --upgrade pip
+
+ # Install packages
+ pip install -r requirements.txt
+ ```
+2. Build
+ > *Run commands in python virtual environment*
+ ```
+
+ # Configure
+ cmake --preset spark
+
+ # Build Debug
+ cmake --build --preset spark-debug
+ dotnet build spark_client.sln --configuration Debug
+
+ # Build Release
+ cmake --build --preset spark-release
+ dotnet build spark_client.sln --configuration Release
+ ```
+
+# Build Sparkd on Linux
+### **Dependencies**
+* **[CMake][cmake]** (version >= 3.23)
+* **[Ninja Build][ninja build]**
+* **[Python][python]** (version >= 3.9)
+* **[Docker][docker]**
+
+### **References**
+* **[CMakePresets][cmake presets]**
+* **[Ninja (CMake generator)][ninja generator]**
+* **[Python Virtual Environment][py venv]**
+* **[Devcontainer](devcontainer)**
+
+### **Prerequisites**
+* Open repository in a VS Code [Devcontainer](../docker/sparkd/dev/README.md), or
+ create your own [docker container](../docker/sparkd/dev/README.md) in which to build
+> *Run commands from the repo root dir unless otherwise noted*
+1. Set up the **[Python virtual environment][py venv]**
+ ```
+ # Create the virtual environment
+ python3 -m venv .venv
+
+ # Activate the virtual environment
+ . .venv/bin/activate
+
+ # Update pip
+ python -m pip install --upgrade pip
+
+ # Install packages
+ pip install -r requirements.txt
+ ```
+2. Build
+ > *Run commands in python virtual environment*
+ ```
+ # Configure clang debug
+ cmake --preset sparkd-clang-debug
+
+ # Build clang debug
+ cmake --build --preset spark-clang-debug
+
+ # Configure gcc release
+ cmake --build --preset sparkd-gcc-release
+
+ # Build gcc release
+ cmake --build --preset sparkd-gcc-release
+ ```
+
+# Build Spark Api Server on Windows
+### **Dependencies**
+* **[CMake][cmake]** (version >= 3.23)
+* **[Python][python]** (version >= 3.9)
+* **[Visual Studio][visual studio]** (version)
+* **[Dotnet][dotnet]** (sdk version >= 7.0.0)
+
+### **References**
+* **[CMakePresets][cmake presets]**
+* **[Visual Studio 17 2022 (CMake generator)][visual studio generator]**
+* **[Python Virtual Environment][py venv]**
+> *Run commands from the repo root dir unless otherwise noted*
+1. Set up the **[Python Virtual Environment][py venv]**
+ ```
+ # Create the virtual environment
+ python3 -m venv .venv
+
+ # Activate the virtual environment
+ .venv\bin\activate.bat
+
+ # Update pip
+ python -m pip install --upgrade pip
+
+ # Install packages
+ pip install -r requirements.txt
+ ```
+2. Build
+ > *Run in python virtual environment*
+ ```
+ # Configure
+ cmake --preset spark-api-server-windows
+
+ # Build debug
+ cmake --build --preset spark-api-server-windows-debug
+ dotnet build Spark.Api.Server.sln --configuration Debug
+
+ # Build release
+ cmake --build --preset spark-api-server-windows-release
+ dotnet build Spark.Api.Server.sln --configuration Release
+ ```
+
+## Linux
+### **Dependencies**
+* **[CMake][cmake]** (version >= 3.23)
+* **[Python][python]** (version >= 3.9)
+* **[Dotnet][dotnet]** (sdk version >= 7.0.0)
+
+### **References**
+* **[CMakePresets][cmake presets]**
+* **[Ninja Multi-Config (CMake generator)][ninja multi-config generator]**
+* **[Python Virtual Environment][py venv]**
+> *Run commands from the repo root dir unless otherwise noted*
+1. Set up the **[Python Virtual Environment][py venv]**
+ ```
+ # Create the virtual environment
+ python3 -m venv .venv
+
+ # Activate the virtual environment
+ . .venv/bin/activate
+
+ # Update pip
+ python -m pip install --upgrade pip
+
+ # Install packages
+ pip install -r requirements.txt
+ ```
+2. Build
+ > *Run in python virtual environment*
+ ```
+ # Configure
+ cmake --preset spark-api-server-linux
+
+ # Build debug
+ cmake --build --preset spark-api-server-linux-debug
+ dotnet build Spark.Api.Server.sln --configuration Debug
+
+ # Build release
+ cmake --build --preset spark-api-server-linux-release
+ dotnet build Spark.Api.Server.sln --configuration Release
+ ```
diff --git a/mdstream_diff.patch b/mdstream_diff.patch
new file mode 100644
index 000000000..89a1139c9
--- /dev/null
+++ b/mdstream_diff.patch
@@ -0,0 +1,66 @@
+diff --git a/src/DataStreamSuite/SharedStreams/Business/MarketData/MDStream.cs b/src/DataStreamSuite/SharedStreams/Business/MarketData/MDStream.cs
+index 643146112..e334c3aba 100644
+--- a/src/DataStreamSuite/SharedStreams/Business/MarketData/MDStream.cs
++++ b/src/DataStreamSuite/SharedStreams/Business/MarketData/MDStream.cs
+@@ -1,7 +1,9 @@
+ namespace BCG {
+ using BCG.Utilities.General;
+
++ using bts;
+ using bts.api;
++ using bts.api.core;
+ using bts.api.domain;
+ using bts.api.synthetic;
+ using bts.md;
+@@ -14,6 +16,7 @@ namespace BCG {
+ using mbts.spark;
+ using mbts.ui;
+
++ using Spark.Api.MarketData;
+ using Spark.Core.Data;
+ using Spark.Core.Disposables;
+ using Spark.Core.Reactive;
+@@ -39,6 +42,8 @@ namespace BCG {
+ new ColumnInfo("sec", 1, typeof(string))}, "md");
+
+ private BlueTable _rows;
++ readonly MarketDataClient mdClient;
++ IDisposable? mdSub;
+
+ public MDStream()
+ : this(String.Empty, ViewStreamDatabase.instance) {
+@@ -49,14 +54,27 @@ namespace BCG {
+ _symbol = symbol;
+ _rows = new BlueTable(_schema);
+ _md_source = new MDSource(TimeSpan.FromSeconds(0.5));
++ mdClient = new(_md_source);
+ }
+
+ protected override void myOnFirstSubscriber() {
+ _vsd.addWorker(this);
+ _md_source.subscribe_securities().Subscribe(c => on_md(c));
++ Log.isnull(mdSub);
++ mdSub = mdClient.books().Subscribe(onBooks);
+ //base.myOnFirstSubscriber();
+ }
+
++ void onBooks(TableUpdate update) {
++ foreach (var book in update.added) {
++ bts.Log.Gen.info($"MDStream.on_md sec: {book.security}");
++ var sec = book.security;
++ IBlueRow r = new BlueRow(_schema, _id++, sec.ToString());
++ _rows.addRow(r);
++ _is_dirty = true;
++ }
++ }
++
+ protected override void myOnLastSubscriber() {
+ //base.myOnLastSubscriber();
+ _vsd.removeWorker(this);
+@@ -82,4 +100,4 @@ namespace BCG {
+ }
+ }
+ }
+-}
+\ No newline at end of file
++}
diff --git a/patches-ohoolihan b/patches-ohoolihan
new file mode 100644
index 000000000..356dc8d8e
--- /dev/null
+++ b/patches-ohoolihan
@@ -0,0 +1,112 @@
+diff --git a/src/client/spark/AW2/VolSurfaceExtensions.cs b/src/client/spark/AW2/VolSurfaceExtensions.cs
+index 6279d82a2..c143902a7 100644
+--- a/src/client/spark/AW2/VolSurfaceExtensions.cs
++++ b/src/client/spark/AW2/VolSurfaceExtensions.cs
+@@ -162,7 +162,8 @@ namespace mbts.spark {
+ api_sec_exp.Exchange.to_venue(),
+ api_sec_exp.SecurityType.contract_case(),
+ api_sec_exp.Symbol,
+- api_sec_exp.Expiration.date_time());
++ api_sec_exp.HasExpiration ?
++ api_sec_exp.Expiration.date_time() : SecurityExpiration.empty.expiration);
+
+ public static SecurityExpiration to_sec_exp(this Security sec) =>
+ to_sec_exp(new SecurityImmutable(sec));
+diff --git a/src/client/spark/Analytics/Core/Settings/SecurityExpiration.cs b/src/client/spark/Analytics/Core/Settings/SecurityExpiration.cs
+index bf9f44455..b2ac80561 100644
+--- a/src/client/spark/Analytics/Core/Settings/SecurityExpiration.cs
++++ b/src/client/spark/Analytics/Core/Settings/SecurityExpiration.cs
+@@ -3,6 +3,7 @@ namespace mbts {
+ using bts.api;
+ using bts.utils;
+
++ using mbts.spark;
+ using mbts.ui.utility;
+ using mbts.utils;
+
+@@ -25,6 +26,9 @@ namespace mbts {
+ SecurityTuple.empty.security_name,
+ DateTime.MinValue);
+
++ public static bts.api.domain.SecurityExpirationImmutable empty_api_sec_expi =>
++ empty.to_api_sec_expi();
++
+ // for deserialization
+ SecurityExpiration() =>
+ security_tuple = SecurityTuple.empty;
+diff --git a/src/client/spark/Analytics/SupportingData/AnalyticsExpirationParameters.cs b/src/client/spark/Analytics/SupportingData/AnalyticsExpirationParameters.cs
+index f5e2a73d6..0ea7b34be 100644
+--- a/src/client/spark/Analytics/SupportingData/AnalyticsExpirationParameters.cs
++++ b/src/client/spark/Analytics/SupportingData/AnalyticsExpirationParameters.cs
+@@ -50,9 +50,7 @@ namespace mbts.analytics {
+ deliverable_security ??= SecurityExpiration.empty;
+ roll_schedule_expiration ??= SecurityExpiration.empty;
+
+- this.driver = driver ?? new(new() {
+- TimeWeightedInterpolated = new()
+- });
++ this.driver = driver ?? _default_driver;
+ _deliverable = deliverable_security.to_api_sec_expi();
+ _roll_schedule = roll_schedule_expiration.to_api_sec_expi();
+ this.roll_target = roll_target;
+@@ -63,10 +61,10 @@ namespace mbts.analytics {
+ }
+
+ public AnalyticsExpirationParameters(ExpirationParameterEntryImmutable db_i) {
+- _security = db_i.SecurityExpiration;
+- _deliverable = db_i.Deliverable;
+- _roll_schedule = db_i.RollSchedule;
+- driver = db_i.Driver;
++ _security = db_i.HasSecurityExpiration ? db_i.SecurityExpiration : _default_api_sec_expi;
++ _deliverable = db_i.HasDeliverable ? db_i.Deliverable : _default_api_sec_expi;
++ _roll_schedule = db_i.HasRollSchedule ? db_i.RollSchedule : _default_api_sec_expi;
++ driver = db_i.HasDriver ? db_i.Driver : _default_driver;
+ roll_target = db_i.RollTarget;
+ riskless_rate = db_i.RisklessRate;
+ settlement_price = db_i.SettlementPrice;
+@@ -74,6 +72,12 @@ namespace mbts.analytics {
+ vega_beta = db_i.VegaBeta;
+ }
+
++ static ExpirationDriverImmutable _default_driver =>
++ new(new() { TimeWeightedInterpolated = new() });
++
++ static bts.api.domain.SecurityExpirationImmutable _default_api_sec_expi =>
++ SecurityExpiration.empty_api_sec_expi;
++
+ public SecurityExpiration security => _security.to_sec_exp();
+ public SecurityExpiration driver_security => driver.HasSecurity ?
+ driver.Security.to_sec_exp() : SecurityExpiration.empty;
+diff --git a/src/client/spark/NMSheets/DataProviders/NMSheetSingleOptionMainDataSource.cs b/src/client/spark/NMSheets/DataProviders/NMSheetSingleOptionMainDataSource.cs
+index 622121388..ec902f142 100644
+--- a/src/client/spark/NMSheets/DataProviders/NMSheetSingleOptionMainDataSource.cs
++++ b/src/client/spark/NMSheets/DataProviders/NMSheetSingleOptionMainDataSource.cs
+@@ -233,9 +233,18 @@ namespace BCG {
+ private IDictionary _get_atm_data(ExpStrikeKeyedData theo_data) {
+ NMSheetSettings sheet_settings = _threaded_bb.read_state(_in_bb_sheet_settings_tag) ?? new NMSheetSettings();
+ VolSurfaceNonVolInput vs_nvinp = _threaded_bb.read_state(_in_exp_param_tag);
+- IDictionary option_exp_roll_data = vs_nvinp != null ?
+- vs_nvinp.get_roll_map((exp) => exp.symbol == _nmsheet_symbol.option_name && exp.security_type == SecurityType.Option).ToDictionary(kv => kv.Key.expiration, kv => kv.Value) :
+- new Dictionary();
++ var option_sym_exp_groups = vs_nvinp
++ ?.get_roll_map(exp =>
++ exp.symbol == _nmsheet_symbol.option_name &&
++ exp.security_type == SecurityType.Option)
++ ?.GroupBy(kv => kv.Key.expiration)
++ ?.ToDictionary(kv => kv.Key, kv => kv.ToList());
++ var option_exp_roll_data = vs_nvinp
++ ?.get_roll_map(exp =>
++ exp.symbol == _nmsheet_symbol.option_name &&
++ exp.security_type == SecurityType.Option)
++ ?.ToDictionary(kv => kv.Key.expiration, kv => kv.Value)
++ ?? new();
+ IDictionary atm_data = OptionATMDataSource.compute_atm_data(_nmsheet_symbol.option_name, _get_onefive(), theo_data, option_exp_roll_data);
+ if (sheet_settings.setting_atm.select_atm_nearest_strike_only)
+ foreach (ATMData exp_atm_data in atm_data.Values)
+@@ -306,4 +315,4 @@ namespace BCG {
+ }
+ #endregion
+ }
+-}
+\ No newline at end of file
++}
diff --git a/simon_private.pem b/simon_private.pem
new file mode 100644
index 000000000..845090da3
--- /dev/null
+++ b/simon_private.pem
@@ -0,0 +1,30 @@
+-----BEGIN ENCRYPTED PRIVATE KEY-----
+MIIFHDBOBgkqhkiG9w0BBQ0wQTApBgkqhkiG9w0BBQwwHAQIgmHCnJpc1awCAggA
+MAwGCCqGSIb3DQIJBQAwFAYIKoZIhvcNAwcECPGPVlg9BOO1BIIEyHTdfD5zGKyf
+hHVm4So+G8TtIQvL+xu0oLNKQR/DVo3wo3iUD70h7oXQnLJ4CL5qo9nR75u2UghL
+mg6xGFbmvpIX/2LAHsS3+MzmOYWKAr0sjmFgJ0YJ/Km6zL88wpk8Gxf3KW3PTZta
++STA1OVPzggSxbHNV16kD9mVUwzOGxD6+zmkRBrvVr3z7/Y8DO8102QrT13U7wy3
+RWWo9cNFSS4UG1yeOy0IGRzvfPVf8si6xO2GXKJZX1W3pfpUihEVDynz9I7pkw0/
+NF9CVROOgY351d0Vs7is+K9q1etpAHTRJl5phpQUXX6DTXlkjWInAXYcs+4z/kEK
+icUa9fijxvz+rx67WkEn4TmaBCIkPVJukeJFh2NHKEj9nK88/okjDExes9TDC4HC
+riGoT7honGjkmeIQd0EZo3YXzCCRcr5hCjx3z5CQThLY0MT3avNqYLs6qwaEL4ho
+Mnkiiwm/d8n5AGpqm1dI8hmJMhlQzL5a6fpcgst6bqVc7MMRrTNnY/k8ptoXyHWI
+XXkcf1SWBxwK5u14AoH5u40LORz0Shn9PKL05EeAqK5eh9TX+HmxCE7pES4uceRJ
+nBYjPp8q/NkhqHhMlVZgNcZoY3edPY4We5nYKpW4uLFl9DPckFKlaQpN+dVcSY0I
+vIsRaKXeju3YpzehRE80ztE4f8ZO8nyuGScx1PV0mYlXsjMJpAkV2J7Y3BVdFRsk
+bvQ2zfhgX5LQeC9MU0jn5cZbBO67eFxmEvtZKRt0ooZSNR748NqbfO2wHOuxfLOe
+UeZfKGvpJhHqH+nZQiEKIldf7A9ORQeVr/yhR+shv91GJtUE89cm725ODFe5NiHw
+FWVgyBFuB4DN/XVvD8oS7S+Io/q+YAfbDzYqqYjlCuoQcrEfhkcWJOKZ7nEXqEdN
+W/Ly6HAl2QddVK8xj93IGLYhzEZSU8ldw7KhRwXE2jAe9cB+tygOWygxfVpt6d3i
+q0xBLvlUVWC2aviOtQ+xvNb54JPPRTnOZ2RsOUkTPDrTEFeEm0O0lbPI1PSIMlGH
+BSQukR+n0vRfmRRs/OXipe2zJEDy6+BJ44uV+Pz5aARH4arvnfWnRKlx5+b58HEd
+Six4y+Sz2i5k6MwRCQyuPTBJHpp6zvwFCX1doPrcCN6Q0GjtDWGklPyofmuLSM3d
+8sHciqGLfv6+hSH/yY/SbyyU2MYx5WJoREYIfQ1M/V6ycppmIb7QDXrQ6nVd33FC
+73OpKaB/q1tIgdImUT7J0PeM6EmjKC3g6y0iGp9adAF2BMe19rOQjrn9lHMbFb9w
+/4uZDTLPmYZyONmMFnL47PoD3MUlJFvDPN//9g1DTk3lMLHETNzv7Lh6vR2h5Otl
+x9SnakCahQDcLYvxdKNSKHfClwMuS3gjpoT45Qki0APH+dYwGYNXDLj41t61fYkl
+v7IoGoHJ3Ym8ySWtS5MMt4kdb7eiVAzGla5TEa1RqOtU31A6W3GKSOzlY7znaBoi
+Q/aGtZq/AhU4DOba5pT/ddB442fnG/8i2un6oSAthtK6VvHx4ebBDxTgV8Fp62le
+dCLw93+lppVS8ChRH2MQDS2Mr/khQo5f+BybnY6zVBRJh7rvQ7g4U0qBzpoaj+nD
+IiX/FNTzdAYjmFa+iHB2OQ==
+-----END ENCRYPTED PRIVATE KEY-----
diff --git a/simon_public.pem b/simon_public.pem
new file mode 100644
index 000000000..54568b023
--- /dev/null
+++ b/simon_public.pem
@@ -0,0 +1,9 @@
+-----BEGIN PUBLIC KEY-----
+MIIBIjANBgkqhkiG9w0BAQEFAAOCAQ8AMIIBCgKCAQEAvvtV/IP1l/F85ME8N8H+
+8TCBOB68Lxd7sdR0geJGssLBq5F22IIlVKylNxiQV3TE6/hmL9dmr8FuUMrjPCal
+vma0axyAdyCMjFH3U0bYzzxnxSWuj5VJIgaZKOVQuX/3Hb+l4RZi7VWZgQFvEJ+F
+y66S2wDu4ey71Mt0yxkMsl5WMZBL3M1z+bt3pKzmBoyIMVLDcV1k1Yr2wspIRriZ
+95er3JWofK1P9s8B3OoqypEopLRW5hJaZN8YRpFNq5VwRUSj83KVobBAQTWDdOA5
+zOkixPVklZFaNyR06iT0gcgZUUM2bjyfQJGQ4GBCZpZb8KPPgtP+WhghoWby9OLv
+KQIDAQAB
+-----END PUBLIC KEY-----
diff --git a/tools/build_protos.cmd b/tools/build_protos.cmd
new file mode 100644
index 000000000..23c789e6e
--- /dev/null
+++ b/tools/build_protos.cmd
@@ -0,0 +1,14 @@
+@setlocal
+
+@REM set HOME=%HOMEDRIVE%\%HOMEPATH%
+@REM set REPO=%HOME
+@REM for /f "usebackq tokens=*" %%a in (`echo Test`) do my_command %%a
+set TOOLS_DIR=%~dp0
+set REPO_DIR=%~dp0..\
+set BUILD_SPARK_PY=%TOOLS_DIR%\py\build_spark.py
+set PROTOC=%REPO_DIR%externals\windows\bin\protoc.exe
+@REM c:\Users\John\Documents\repos\bts\spark\externals\windows\bin\protoc.exe C:\Users\John\Documents\repos\bts\spark\src\lib\config\protos\data_stream_config.proto --error_format=msvs --proto_path
+@REM =c:\Users\John\Documents\repos\bts\spark\src --plugin="protoc-gen-spark-csharp=c:\Users\John\Documents\repos\bts\spark\src\lib\api\service_compiler\plugin.cmd" --spark-csharp_out=lang=c#,templates=immutable.pb.cs,strip_out_path=1:c:\Users\John\Documents\repos\bts\spark\build\Debug\obj\DataStrea
+@REM mCore/lib/config/protos --csharp_out=c:\Users\John\Documents\repos\bts\spark\build\Debug\obj\DataStreamCore/lib/config/protos"
+
+%PROTOC% %REPO_DIR%src\lib\config\protos\data_stream_config.proto --error_format=msvs --proto_path=%REPO_DIR%src --plugin="protoc-gen-spark-csharp=%REPO_DIR%src\lib\api\service_compiler\plugin.cmd" --spark-csharp_out=lang=c#,templates=immutable.pb.cs,strip_out_path=1:%REPO_DIR%build\Debug\obj\DataStreamCore\lib\config\protos --csharp_out=%REPO_DIR%build\Debug\obj\DataStreamCore\lib\config\protos
diff --git a/tools/pack_sparkd.sh b/tools/pack_sparkd.sh
new file mode 100644
index 000000000..17d70b1e1
--- /dev/null
+++ b/tools/pack_sparkd.sh
@@ -0,0 +1,85 @@
+#!/usr/bin/env bash
+
+function _exit() {
+ if [[ "$#" -gt 1 ]]; then
+ # When there are multiple args, the last arg is the exit code.
+ echo "${@:1:($# - 1)}"
+ exit "${@: -1}"
+ fi
+ echo "$1"
+ exit 0
+}
+
+help_text="\
+Package a self-contained sparkd executable in a tar.gz file
+
+Usage: pack_sparkd.sh [OPTIONS]
+
+Positionals:
+ OUT The output path for the resultant sparkd tar.gz
+
+Options:
+ --out OUT The output path for the resultant sparkd tar.gz
+ --force Overwrite existing sparkd tar.gz
+ -h,--help Print this help message and exit
+
+"
+
+dest=""
+pkg_name=""
+declare -i overwrite_dest=0
+
+while [[ $# -gt 0 ]]; do
+ case "$1" in
+ --out)
+ if [[ -z "$2" ]]; then
+ _exit "OUT (--out) must be non-empty" 1
+ fi
+ dest="$2"
+ shift 2
+ ;;
+ --force)
+ overwrite_dest=1
+ shift 1
+ ;;
+ -h)
+ ;&
+ --help)
+ _exit "$help_text" 0
+ ;;
+ *)
+ esac
+done
+
+tools_dir="$(dirname "$(readlink -f ${BASH_SOURCE[0]})")"
+repo_dir="$tools_dir/.."
+build_dir="$repo_dir/build/gcc/release"
+install_dir="$repo_dir/install/Release/"
+revision=$(git rev-parse --short HEAD)
+timestamp=$(date +'%Y-%m%d-%H%M')
+
+if [[ -z "$dest" ]]; then
+ pkg_name="sparkd_${revision}_${timestamp}.tar.gz"
+ dest="$repo_dir/$pkg_name"
+else
+ pkg_name="$(basename "$dest")"
+fi
+
+if [ -e "$dest" ]; then
+ echo "The output file '$dest' already exists (use --force to overwrite)"
+ exit 1
+fi
+eval "\
+ $tools_dir/build_sparkd.sh --compiler gcc --build-type release && \
+ cmake --build $build_dir --target install"
+if [[ $? -ne 0 ]]; then
+ exit $?
+fi
+cd "$install_dir"
+echo "Creating '$pkg_name'"
+tar -czf "$pkg_name" "sparkd"
+if [[ $? -ne 0 ]]; then
+ exit $?
+fi
+
+mv "$pkg_name" "$dest"
diff --git a/tools/py/gen_cmd_aliases.py b/tools/py/gen_cmd_aliases.py
new file mode 100644
index 000000000..63891a9d7
--- /dev/null
+++ b/tools/py/gen_cmd_aliases.py
@@ -0,0 +1,79 @@
+import argparse
+import os
+from pathlib import Path
+import subprocess
+
+from core.file_utils import SparkPaths, create_dir, find_files, find_file_tree
+from core.iter_utils import at_bounded
+from core.logging import exit_msg
+
+
+def _get_arg_parser():
+ parser = argparse.ArgumentParser(
+ description='Generates cmd scripts (aliases) for py scripts')
+ parser.add_argument(
+ '--sources',
+ nargs='+',
+ help='The source files for which to generate cs',
+ )
+ parser.add_argument(
+ '--out-dir',
+ metavar='PATH',
+ help='The directory in which to dump the aliases',
+ )
+ parser.add_argument(
+ '--python-exe',
+ metavar='PATH',
+ help='The python exe to use in the cmd aliases',
+ )
+ parser.add_argument(
+ '--clean',
+ action='store_true',
+ help='Delete existing lexer.cs and parser.cs files in --out-dir',
+ )
+ parser.add_argument(
+ '--overwrite-existing',
+ action='store_true',
+ help=
+ 'Overwrite existing files with the generated aliases (if necessary)',
+ )
+ parser.add_argument(
+ '--recursive',
+ help='Whether or not to generate aliases recursively',
+ )
+ this_dir = Path(__file__).parent.absolute()
+ parser.set_defaults(
+ sources=[this_dir],
+ python_exe='python',
+ clean=False,
+ )
+ return parser
+
+
+def _main():
+ parser = _get_arg_parser()
+ args = parser.parse_args()
+ if args.sources is None:
+ print('No sources specified')
+ sources = [Path(s).absolute() for s in args.sources]
+ scripts = [f.path for s in sources for f in find_file_tree(s, '.py', 1)]
+ static_out_dir = None if args.out_dir is None else Path(args.out_dir)
+
+ for script in scripts:
+ out_dir = static_out_dir or script.parent.with_name(
+ f'{script.parent.name}.aliases')
+ create_dir(out_dir)
+ alias = out_dir.joinpath(f'{script.stem}.cmd')
+ if alias.exists():
+ print(f'Skipping {alias} because it already exists')
+ continue
+ print(f'Generating {alias}')
+ with open(alias, 'w') as alias_out:
+ alias_out.write(f'@{args.python_exe} {script.absolute()} %*\n')
+
+
+if __name__ == "__main__":
+ try:
+ _main()
+ except Exception as e:
+ print(f'gen_cmd_aliases.py failed: {e}')
diff --git a/tools/spark_dev_prompt.bat b/tools/spark_dev_prompt.bat
new file mode 100644
index 000000000..300ac5e7b
--- /dev/null
+++ b/tools/spark_dev_prompt.bat
@@ -0,0 +1 @@
+@"C:\\Program Files\\Microsoft Visual Studio\\2022\\Community\\Common7\\Tools\\VsDevCmd.bat" -arch=x64 -host_arch=x64 && %~dp0..\\.venv\\Scripts\\activate.bat