diff --git a/KeyCode.conf b/KeyCode.conf new file mode 100644 index 000000000..5ac45f897 --- /dev/null +++ b/KeyCode.conf @@ -0,0 +1,15 @@ +#nusers: +#allow concurrent : true +#expires: 2024-Oct-06 00:00:00 638637696000000000 +#generation_time: 2023-Oct-06 18:58:48.015723 5250008173707545134 +EncryptedKeyCode { + value { + keycode: "KgoMeiVPzf/kv65GYGCyQQJVY/11sId6LYRwNNlNwA83+sISiSQnOZrziSgxjEAcaAZQn1nV4hY6MRi4XXznvxLYnDKf2bK0yktWWpSSqxqNxbDQ6AVSk5MGHptbsQSGYpYHlyCi0Niv8ES5NTj/FiBJXPrGEZABZBbNkqXIOa0HDdcxY+joeIf32ekX+5/LXN5XDTZBFT2A7q+IPDrFbKzSBvlzGOONSlq+XyKZZPqt0rXGO45V/GxFUTMYiEvn1/72iPGQ715tFXkr9ozGtitonnixzvW8Sfwmu7vM8VpWfoAc9D/1w5m5Ol5BS+Je8wigQysyrC8rud2KU+f7bQ==" + } +} + +PublicKey { + value { + key: "-----BEGIN PUBLIC KEY-----\nMIIBIjANBgkqhkiG9w0BAQEFAAOCAQ8AMIIBCgKCAQEAvvtV/IP1l/F85ME8N8H+\n8TCBOB68Lxd7sdR0geJGssLBq5F22IIlVKylNxiQV3TE6/hmL9dmr8FuUMrjPCal\nvma0axyAdyCMjFH3U0bYzzxnxSWuj5VJIgaZKOVQuX/3Hb+l4RZi7VWZgQFvEJ+F\ny66S2wDu4ey71Mt0yxkMsl5WMZBL3M1z+bt3pKzmBoyIMVLDcV1k1Yr2wspIRriZ\n95er3JWofK1P9s8B3OoqypEopLRW5hJaZN8YRpFNq5VwRUSj83KVobBAQTWDdOA5\nzOkixPVklZFaNyR06iT0gcgZUUM2bjyfQJGQ4GBCZpZb8KPPgtP+WhghoWby9OLv\nKQIDAQAB\n-----END PUBLIC KEY-----" + } +} diff --git a/cfgl b/cfgl new file mode 100644 index 000000000..81dff3389 --- /dev/null +++ b/cfgl @@ -0,0 +1,33 @@ +diff.astextplain.textconv=astextplain +filter.lfs.clean=git-lfs clean -- %f +filter.lfs.smudge=git-lfs smudge -- %f +filter.lfs.process=git-lfs filter-process +filter.lfs.required=true +http.sslbackend=openssl +http.sslcainfo=C:/Program Files/Git/mingw64/etc/ssl/certs/ca-bundle.crt +core.autocrlf=true +core.fscache=true +core.symlinks=false +pull.rebase=false +credential.helper=manager +credential.https://dev.azure.com.usehttppath=true +init.defaultbranch=master +status.short=true +core.repositoryformatversion=0 +core.filemode=false +core.bare=false +core.logallrefupdates=true +core.symlinks=false +core.ignorecase=true +submodule.active=. +remote.origin.url=git@bitbucket.org:bluetradesys/spark.git +remote.origin.fetch=+refs/heads/*:refs/remotes/origin/* +branch.spark_crypto.dev.remote=origin +branch.spark_crypto.dev.merge=refs/heads/spark_crypto.dev +submodule.docker/linux.url=git@bitbucket.org:bluetradesys/linux_docker.git +submodule.docker/windows.url=git@bitbucket.org:bluetradesys/windows_docker.git +submodule.externals.url=git@bitbucket.org:bluetradesys/externals.git +branch.development.remote=origin +branch.development.merge=refs/heads/development +user.name=John Colvin +user.email=john.colvin@bluetradesys.com diff --git a/get-token.py b/get-token.py new file mode 100644 index 000000000..ee22fcaaa --- /dev/null +++ b/get-token.py @@ -0,0 +1,45 @@ +# import json +# from rauth import OAuth2Service + + +# class ExampleOAuth2Client: +# def __init__(self, url, path, client_id, client_secret): +# self.access_token = None +# self.service = OAuth2Service( +# name="foo", +# client_id=client_id, +# client_secret=client_secret, +# access_token_url=f'{url}{path}', +# authorize_url=f'{url}{path}', +# base_url=url, +# ) +# self.get_access_token() + +# def get_access_token(self): +# data = { +# 'grant_type': 'client_credentials', # generally required! +# } +# session = self.service.get_auth_session(data=data, decoder=json.loads) +# self.access_token = session.access_token + + +# def _main(): +# client_id = 'API_THINKALPHA123' +# client_secret = '$MM9Rn*#qYARY$$QTMA#AMn*' +# oath = ExampleOAuth2Client( +# url='https://authnr.cmegroup.com', +# path='/as/token.oath2', +# client_id=client_id, +# client_secret=client_secret, +# ) +# tok = oath.get_access_token() +# print(tok) + + +# if __name__ == '__main__': +# try: +# _main() +# except json.JSONDecodeError as jde: +# print(f'get-token.py failed with json-decode-error: {jde}.\n{jde.doc}\n\nlineno: {jde.lineno}\npos: {jde.pos}') +# except Exception as e: +# print(f'get-token.py failed: {e}') diff --git a/get-token2.py b/get-token2.py new file mode 100644 index 000000000..64813dce2 --- /dev/null +++ b/get-token2.py @@ -0,0 +1,30 @@ +import subprocess + + +def _main(): + host = "markets.api.uat.cmegroup.com" + port = "443" + # AUTH_ENDPOINT="$host/marketdatastream/v1" + AUTH_ENDPOINT = 'https://authnr.cmegroup.com/as/token.oauth2' + API_KEY = "API_THINKALPHA123" + API_SECRET = '$MM9Rn*#qYARY$$QTMA#AMn*' + + # Define your API key and secret + # API_KEY="YOUR_API_KEY" + # API_SECRET="YOUR_API_SECRET" + + # Define the authentication API endpoint URL + # AUTH_ENDPOINT="https://api.example.com/auth" + + # Perform the POST request and capture the response + # response=$(curl -s -X POST -d "api_key=$API_KEY&api_secret=$API_SECRET&grant_type=password&username=API_THINKALPHA123&password=$API_SECRET" "$AUTH_ENDPOINT") + # subprocess.run('ls') + subprocess.run( + f'curl -s -X POST -d "grant_type=client_credentials&client_id={API_KEY}&client_secret={API_SECRET}" "{AUTH_ENDPOINT}"', + ) + + +# curl --user 'API_THINKALPHA123:$MM9Rn*#qYARY$$QTMA#AMn*' 'https://query-api.datamine.cmegroup.com/v3/query/query_order_record?tradedate=2023-07-10&fields=symbol,transacttime,instrumentsequencenumber,tradeprice,tradequantity,tradeflag&q=asset.eq.TN,securitytype.eq.FUT,tradeflag.nin.OBS&fileFormat=csv&securityexchange=XCBT&underlyingproduct=Interest%20Rate' + +if __name__ == '__main__': + _main() diff --git a/get-token3.py b/get-token3.py new file mode 100644 index 000000000..e69de29bb diff --git a/good-dev-readme.md b/good-dev-readme.md new file mode 100644 index 000000000..a3c506609 --- /dev/null +++ b/good-dev-readme.md @@ -0,0 +1,211 @@ +[cmake]: https://cmake.org/ +[cmake ext]: https://marketplace.visualstudio.com/items?itemName=ms-vscode.cmake-tools +[cmake presets]: https://cmake.org/cmake/help/latest/manual/cmake-presets.7.html +[clangd ext]: https://marketplace.visualstudio.com/items?itemName=llvm-vs-code-extensions.vscode-clangd +[cpp ext]: https://marketplace.visualstudio.com/items?itemName=ms-vscode.cpptools +[docker]: https://www.docker.com/ +[dotnet]: https://dotnet.microsoft.com/en-us/download/dotnet +[ninja build]: https://ninja-build.org/ +[ninja generator]: https://cmake.org/cmake/help/latest/generator/Ninja.html +[ninja multi-config generator]: https://cmake.org/cmake/help/latest/generator/Ninja%20Multi-Config.html +[python]: https://www.python.org/ +[py venv]: https://docs.python.org/3/library/venv.html +[visual studio]: https://visualstudio.microsoft.com/ +[visual studio generator]: https://cmake.org/cmake/help/latest/generator/Visual%20Studio%2017%202022.html +[devcontainer]: https://code.visualstudio.com/docs/devcontainers/create-dev-container + +# C++ Intellisense for Spark on Windows (VS Code) +**Steps** +1. Install the **[C++ extension][cpp ext]** +2. Install the **[CMake extension][cmake ext]** +3. Copy `c_cpp_properties.json` from this folder to `$workspaceFolder/.vscode/` +4. In the VS Code Command Palette, run `CMake: Select Configure Preset`, then + select '**spark**' from the dropdown + +**Notes** +* Intellisense won't work for generated files/types until you build libspark + +# C++ Intellisense for Sparkd on Linux (VS Code) +1. Install the **[CMake extension][cmake ext]** +2. Install the **[Clangd extension][clangd ext]** +3. (Optional) Install the **[C++ extension][cpp ext]** (*needed for debugging, + but intellisense should work without it*). +4. If the **[C++ extension][cpp ext]** is installed, disable its intellisense in + your workspace settings (*Clangd intellisense is better*) + ``` + "C_Cpp.intelliSenseEngine": "disabled" + ``` +5. In the VS Code Command Palette, run **CMake: Select Configure Preset**, then select the + **[preset][cmake presets]** you wish to build from the dropdown + * *The **Clang** presets may work best with the **Clangd** extension* + * *Intellisense won't work for generated files/types until you build libspark* + +# Build Spark on Windows +### **Dependencies** +* **[CMake][cmake]** (version >= 3.23) +* **[Python][python]** (version >= 3.9) +* **[Visual Studio][visual studio]** (2022) +* **[Dotnet][dotnet]** (sdk version >= 7.0.0) + +### **References** +* **[CMakePresets][cmake presets]** +* **[Visual Studio 17 2022 (CMake generator)][visual studio generator]** +* **[Python Virtual Environment][py venv]** +> *Run commands from the repo root dir unless otherwise noted* +1. Set up the **[Python virtual environment][py venv]**
+ ``` + # Create the virtual environment + python3 -m venv .venv + + # Activate the virtual environment + .venv\bin\activate.bat + + # Update pip + python -m pip install --upgrade pip + + # Install packages + pip install -r requirements.txt + ``` +2. Build + > *Run commands in python virtual environment* + ``` + + # Configure + cmake --preset spark + + # Build Debug + cmake --build --preset spark-debug + dotnet build spark_client.sln --configuration Debug + + # Build Release + cmake --build --preset spark-release + dotnet build spark_client.sln --configuration Release + ``` + +# Build Sparkd on Linux +### **Dependencies** +* **[CMake][cmake]** (version >= 3.23) +* **[Ninja Build][ninja build]** +* **[Python][python]** (version >= 3.9) +* **[Docker][docker]** + +### **References** +* **[CMakePresets][cmake presets]** +* **[Ninja (CMake generator)][ninja generator]** +* **[Python Virtual Environment][py venv]** +* **[Devcontainer](devcontainer)** + +### **Prerequisites** +* Open repository in a VS Code [Devcontainer](../docker/sparkd/dev/README.md), or + create your own [docker container](../docker/sparkd/dev/README.md) in which to build +> *Run commands from the repo root dir unless otherwise noted*
+1. Set up the **[Python virtual environment][py venv]** + ``` + # Create the virtual environment + python3 -m venv .venv + + # Activate the virtual environment + . .venv/bin/activate + + # Update pip + python -m pip install --upgrade pip + + # Install packages + pip install -r requirements.txt + ``` +2. Build + > *Run commands in python virtual environment* + ``` + # Configure clang debug + cmake --preset sparkd-clang-debug + + # Build clang debug + cmake --build --preset spark-clang-debug + + # Configure gcc release + cmake --build --preset sparkd-gcc-release + + # Build gcc release + cmake --build --preset sparkd-gcc-release + ``` + +# Build Spark Api Server on Windows +### **Dependencies** +* **[CMake][cmake]** (version >= 3.23) +* **[Python][python]** (version >= 3.9) +* **[Visual Studio][visual studio]** (version) +* **[Dotnet][dotnet]** (sdk version >= 7.0.0) + +### **References** +* **[CMakePresets][cmake presets]** +* **[Visual Studio 17 2022 (CMake generator)][visual studio generator]** +* **[Python Virtual Environment][py venv]** +> *Run commands from the repo root dir unless otherwise noted* +1. Set up the **[Python Virtual Environment][py venv]** + ``` + # Create the virtual environment + python3 -m venv .venv + + # Activate the virtual environment + .venv\bin\activate.bat + + # Update pip + python -m pip install --upgrade pip + + # Install packages + pip install -r requirements.txt + ``` +2. Build + > *Run in python virtual environment* + ``` + # Configure + cmake --preset spark-api-server-windows + + # Build debug + cmake --build --preset spark-api-server-windows-debug + dotnet build Spark.Api.Server.sln --configuration Debug + + # Build release + cmake --build --preset spark-api-server-windows-release + dotnet build Spark.Api.Server.sln --configuration Release + ``` + +## Linux +### **Dependencies** +* **[CMake][cmake]** (version >= 3.23) +* **[Python][python]** (version >= 3.9) +* **[Dotnet][dotnet]** (sdk version >= 7.0.0) + +### **References** +* **[CMakePresets][cmake presets]** +* **[Ninja Multi-Config (CMake generator)][ninja multi-config generator]** +* **[Python Virtual Environment][py venv]** +> *Run commands from the repo root dir unless otherwise noted* +1. Set up the **[Python Virtual Environment][py venv]** + ``` + # Create the virtual environment + python3 -m venv .venv + + # Activate the virtual environment + . .venv/bin/activate + + # Update pip + python -m pip install --upgrade pip + + # Install packages + pip install -r requirements.txt + ``` +2. Build + > *Run in python virtual environment* + ``` + # Configure + cmake --preset spark-api-server-linux + + # Build debug + cmake --build --preset spark-api-server-linux-debug + dotnet build Spark.Api.Server.sln --configuration Debug + + # Build release + cmake --build --preset spark-api-server-linux-release + dotnet build Spark.Api.Server.sln --configuration Release + ``` diff --git a/mdstream_diff.patch b/mdstream_diff.patch new file mode 100644 index 000000000..89a1139c9 --- /dev/null +++ b/mdstream_diff.patch @@ -0,0 +1,66 @@ +diff --git a/src/DataStreamSuite/SharedStreams/Business/MarketData/MDStream.cs b/src/DataStreamSuite/SharedStreams/Business/MarketData/MDStream.cs +index 643146112..e334c3aba 100644 +--- a/src/DataStreamSuite/SharedStreams/Business/MarketData/MDStream.cs ++++ b/src/DataStreamSuite/SharedStreams/Business/MarketData/MDStream.cs +@@ -1,7 +1,9 @@ + namespace BCG { + using BCG.Utilities.General; + ++ using bts; + using bts.api; ++ using bts.api.core; + using bts.api.domain; + using bts.api.synthetic; + using bts.md; +@@ -14,6 +16,7 @@ namespace BCG { + using mbts.spark; + using mbts.ui; + ++ using Spark.Api.MarketData; + using Spark.Core.Data; + using Spark.Core.Disposables; + using Spark.Core.Reactive; +@@ -39,6 +42,8 @@ namespace BCG { + new ColumnInfo("sec", 1, typeof(string))}, "md"); + + private BlueTable _rows; ++ readonly MarketDataClient mdClient; ++ IDisposable? mdSub; + + public MDStream() + : this(String.Empty, ViewStreamDatabase.instance) { +@@ -49,14 +54,27 @@ namespace BCG { + _symbol = symbol; + _rows = new BlueTable(_schema); + _md_source = new MDSource(TimeSpan.FromSeconds(0.5)); ++ mdClient = new(_md_source); + } + + protected override void myOnFirstSubscriber() { + _vsd.addWorker(this); + _md_source.subscribe_securities().Subscribe(c => on_md(c)); ++ Log.isnull(mdSub); ++ mdSub = mdClient.books().Subscribe(onBooks); + //base.myOnFirstSubscriber(); + } + ++ void onBooks(TableUpdate update) { ++ foreach (var book in update.added) { ++ bts.Log.Gen.info($"MDStream.on_md sec: {book.security}"); ++ var sec = book.security; ++ IBlueRow r = new BlueRow(_schema, _id++, sec.ToString()); ++ _rows.addRow(r); ++ _is_dirty = true; ++ } ++ } ++ + protected override void myOnLastSubscriber() { + //base.myOnLastSubscriber(); + _vsd.removeWorker(this); +@@ -82,4 +100,4 @@ namespace BCG { + } + } + } +-} +\ No newline at end of file ++} diff --git a/patches-ohoolihan b/patches-ohoolihan new file mode 100644 index 000000000..356dc8d8e --- /dev/null +++ b/patches-ohoolihan @@ -0,0 +1,112 @@ +diff --git a/src/client/spark/AW2/VolSurfaceExtensions.cs b/src/client/spark/AW2/VolSurfaceExtensions.cs +index 6279d82a2..c143902a7 100644 +--- a/src/client/spark/AW2/VolSurfaceExtensions.cs ++++ b/src/client/spark/AW2/VolSurfaceExtensions.cs +@@ -162,7 +162,8 @@ namespace mbts.spark { + api_sec_exp.Exchange.to_venue(), + api_sec_exp.SecurityType.contract_case(), + api_sec_exp.Symbol, +- api_sec_exp.Expiration.date_time()); ++ api_sec_exp.HasExpiration ? ++ api_sec_exp.Expiration.date_time() : SecurityExpiration.empty.expiration); + + public static SecurityExpiration to_sec_exp(this Security sec) => + to_sec_exp(new SecurityImmutable(sec)); +diff --git a/src/client/spark/Analytics/Core/Settings/SecurityExpiration.cs b/src/client/spark/Analytics/Core/Settings/SecurityExpiration.cs +index bf9f44455..b2ac80561 100644 +--- a/src/client/spark/Analytics/Core/Settings/SecurityExpiration.cs ++++ b/src/client/spark/Analytics/Core/Settings/SecurityExpiration.cs +@@ -3,6 +3,7 @@ namespace mbts { + using bts.api; + using bts.utils; + ++ using mbts.spark; + using mbts.ui.utility; + using mbts.utils; + +@@ -25,6 +26,9 @@ namespace mbts { + SecurityTuple.empty.security_name, + DateTime.MinValue); + ++ public static bts.api.domain.SecurityExpirationImmutable empty_api_sec_expi => ++ empty.to_api_sec_expi(); ++ + // for deserialization + SecurityExpiration() => + security_tuple = SecurityTuple.empty; +diff --git a/src/client/spark/Analytics/SupportingData/AnalyticsExpirationParameters.cs b/src/client/spark/Analytics/SupportingData/AnalyticsExpirationParameters.cs +index f5e2a73d6..0ea7b34be 100644 +--- a/src/client/spark/Analytics/SupportingData/AnalyticsExpirationParameters.cs ++++ b/src/client/spark/Analytics/SupportingData/AnalyticsExpirationParameters.cs +@@ -50,9 +50,7 @@ namespace mbts.analytics { + deliverable_security ??= SecurityExpiration.empty; + roll_schedule_expiration ??= SecurityExpiration.empty; + +- this.driver = driver ?? new(new() { +- TimeWeightedInterpolated = new() +- }); ++ this.driver = driver ?? _default_driver; + _deliverable = deliverable_security.to_api_sec_expi(); + _roll_schedule = roll_schedule_expiration.to_api_sec_expi(); + this.roll_target = roll_target; +@@ -63,10 +61,10 @@ namespace mbts.analytics { + } + + public AnalyticsExpirationParameters(ExpirationParameterEntryImmutable db_i) { +- _security = db_i.SecurityExpiration; +- _deliverable = db_i.Deliverable; +- _roll_schedule = db_i.RollSchedule; +- driver = db_i.Driver; ++ _security = db_i.HasSecurityExpiration ? db_i.SecurityExpiration : _default_api_sec_expi; ++ _deliverable = db_i.HasDeliverable ? db_i.Deliverable : _default_api_sec_expi; ++ _roll_schedule = db_i.HasRollSchedule ? db_i.RollSchedule : _default_api_sec_expi; ++ driver = db_i.HasDriver ? db_i.Driver : _default_driver; + roll_target = db_i.RollTarget; + riskless_rate = db_i.RisklessRate; + settlement_price = db_i.SettlementPrice; +@@ -74,6 +72,12 @@ namespace mbts.analytics { + vega_beta = db_i.VegaBeta; + } + ++ static ExpirationDriverImmutable _default_driver => ++ new(new() { TimeWeightedInterpolated = new() }); ++ ++ static bts.api.domain.SecurityExpirationImmutable _default_api_sec_expi => ++ SecurityExpiration.empty_api_sec_expi; ++ + public SecurityExpiration security => _security.to_sec_exp(); + public SecurityExpiration driver_security => driver.HasSecurity ? + driver.Security.to_sec_exp() : SecurityExpiration.empty; +diff --git a/src/client/spark/NMSheets/DataProviders/NMSheetSingleOptionMainDataSource.cs b/src/client/spark/NMSheets/DataProviders/NMSheetSingleOptionMainDataSource.cs +index 622121388..ec902f142 100644 +--- a/src/client/spark/NMSheets/DataProviders/NMSheetSingleOptionMainDataSource.cs ++++ b/src/client/spark/NMSheets/DataProviders/NMSheetSingleOptionMainDataSource.cs +@@ -233,9 +233,18 @@ namespace BCG { + private IDictionary _get_atm_data(ExpStrikeKeyedData theo_data) { + NMSheetSettings sheet_settings = _threaded_bb.read_state(_in_bb_sheet_settings_tag) ?? new NMSheetSettings(); + VolSurfaceNonVolInput vs_nvinp = _threaded_bb.read_state(_in_exp_param_tag); +- IDictionary option_exp_roll_data = vs_nvinp != null ? +- vs_nvinp.get_roll_map((exp) => exp.symbol == _nmsheet_symbol.option_name && exp.security_type == SecurityType.Option).ToDictionary(kv => kv.Key.expiration, kv => kv.Value) : +- new Dictionary(); ++ var option_sym_exp_groups = vs_nvinp ++ ?.get_roll_map(exp => ++ exp.symbol == _nmsheet_symbol.option_name && ++ exp.security_type == SecurityType.Option) ++ ?.GroupBy(kv => kv.Key.expiration) ++ ?.ToDictionary(kv => kv.Key, kv => kv.ToList()); ++ var option_exp_roll_data = vs_nvinp ++ ?.get_roll_map(exp => ++ exp.symbol == _nmsheet_symbol.option_name && ++ exp.security_type == SecurityType.Option) ++ ?.ToDictionary(kv => kv.Key.expiration, kv => kv.Value) ++ ?? new(); + IDictionary atm_data = OptionATMDataSource.compute_atm_data(_nmsheet_symbol.option_name, _get_onefive(), theo_data, option_exp_roll_data); + if (sheet_settings.setting_atm.select_atm_nearest_strike_only) + foreach (ATMData exp_atm_data in atm_data.Values) +@@ -306,4 +315,4 @@ namespace BCG { + } + #endregion + } +-} +\ No newline at end of file ++} diff --git a/simon_private.pem b/simon_private.pem new file mode 100644 index 000000000..845090da3 --- /dev/null +++ b/simon_private.pem @@ -0,0 +1,30 @@ +-----BEGIN ENCRYPTED PRIVATE KEY----- +MIIFHDBOBgkqhkiG9w0BBQ0wQTApBgkqhkiG9w0BBQwwHAQIgmHCnJpc1awCAggA +MAwGCCqGSIb3DQIJBQAwFAYIKoZIhvcNAwcECPGPVlg9BOO1BIIEyHTdfD5zGKyf +hHVm4So+G8TtIQvL+xu0oLNKQR/DVo3wo3iUD70h7oXQnLJ4CL5qo9nR75u2UghL +mg6xGFbmvpIX/2LAHsS3+MzmOYWKAr0sjmFgJ0YJ/Km6zL88wpk8Gxf3KW3PTZta ++STA1OVPzggSxbHNV16kD9mVUwzOGxD6+zmkRBrvVr3z7/Y8DO8102QrT13U7wy3 +RWWo9cNFSS4UG1yeOy0IGRzvfPVf8si6xO2GXKJZX1W3pfpUihEVDynz9I7pkw0/ +NF9CVROOgY351d0Vs7is+K9q1etpAHTRJl5phpQUXX6DTXlkjWInAXYcs+4z/kEK +icUa9fijxvz+rx67WkEn4TmaBCIkPVJukeJFh2NHKEj9nK88/okjDExes9TDC4HC +riGoT7honGjkmeIQd0EZo3YXzCCRcr5hCjx3z5CQThLY0MT3avNqYLs6qwaEL4ho +Mnkiiwm/d8n5AGpqm1dI8hmJMhlQzL5a6fpcgst6bqVc7MMRrTNnY/k8ptoXyHWI +XXkcf1SWBxwK5u14AoH5u40LORz0Shn9PKL05EeAqK5eh9TX+HmxCE7pES4uceRJ +nBYjPp8q/NkhqHhMlVZgNcZoY3edPY4We5nYKpW4uLFl9DPckFKlaQpN+dVcSY0I +vIsRaKXeju3YpzehRE80ztE4f8ZO8nyuGScx1PV0mYlXsjMJpAkV2J7Y3BVdFRsk +bvQ2zfhgX5LQeC9MU0jn5cZbBO67eFxmEvtZKRt0ooZSNR748NqbfO2wHOuxfLOe +UeZfKGvpJhHqH+nZQiEKIldf7A9ORQeVr/yhR+shv91GJtUE89cm725ODFe5NiHw +FWVgyBFuB4DN/XVvD8oS7S+Io/q+YAfbDzYqqYjlCuoQcrEfhkcWJOKZ7nEXqEdN +W/Ly6HAl2QddVK8xj93IGLYhzEZSU8ldw7KhRwXE2jAe9cB+tygOWygxfVpt6d3i +q0xBLvlUVWC2aviOtQ+xvNb54JPPRTnOZ2RsOUkTPDrTEFeEm0O0lbPI1PSIMlGH +BSQukR+n0vRfmRRs/OXipe2zJEDy6+BJ44uV+Pz5aARH4arvnfWnRKlx5+b58HEd +Six4y+Sz2i5k6MwRCQyuPTBJHpp6zvwFCX1doPrcCN6Q0GjtDWGklPyofmuLSM3d +8sHciqGLfv6+hSH/yY/SbyyU2MYx5WJoREYIfQ1M/V6ycppmIb7QDXrQ6nVd33FC +73OpKaB/q1tIgdImUT7J0PeM6EmjKC3g6y0iGp9adAF2BMe19rOQjrn9lHMbFb9w +/4uZDTLPmYZyONmMFnL47PoD3MUlJFvDPN//9g1DTk3lMLHETNzv7Lh6vR2h5Otl +x9SnakCahQDcLYvxdKNSKHfClwMuS3gjpoT45Qki0APH+dYwGYNXDLj41t61fYkl +v7IoGoHJ3Ym8ySWtS5MMt4kdb7eiVAzGla5TEa1RqOtU31A6W3GKSOzlY7znaBoi +Q/aGtZq/AhU4DOba5pT/ddB442fnG/8i2un6oSAthtK6VvHx4ebBDxTgV8Fp62le +dCLw93+lppVS8ChRH2MQDS2Mr/khQo5f+BybnY6zVBRJh7rvQ7g4U0qBzpoaj+nD +IiX/FNTzdAYjmFa+iHB2OQ== +-----END ENCRYPTED PRIVATE KEY----- diff --git a/simon_public.pem b/simon_public.pem new file mode 100644 index 000000000..54568b023 --- /dev/null +++ b/simon_public.pem @@ -0,0 +1,9 @@ +-----BEGIN PUBLIC KEY----- +MIIBIjANBgkqhkiG9w0BAQEFAAOCAQ8AMIIBCgKCAQEAvvtV/IP1l/F85ME8N8H+ +8TCBOB68Lxd7sdR0geJGssLBq5F22IIlVKylNxiQV3TE6/hmL9dmr8FuUMrjPCal +vma0axyAdyCMjFH3U0bYzzxnxSWuj5VJIgaZKOVQuX/3Hb+l4RZi7VWZgQFvEJ+F +y66S2wDu4ey71Mt0yxkMsl5WMZBL3M1z+bt3pKzmBoyIMVLDcV1k1Yr2wspIRriZ +95er3JWofK1P9s8B3OoqypEopLRW5hJaZN8YRpFNq5VwRUSj83KVobBAQTWDdOA5 +zOkixPVklZFaNyR06iT0gcgZUUM2bjyfQJGQ4GBCZpZb8KPPgtP+WhghoWby9OLv +KQIDAQAB +-----END PUBLIC KEY----- diff --git a/tools/build_protos.cmd b/tools/build_protos.cmd new file mode 100644 index 000000000..23c789e6e --- /dev/null +++ b/tools/build_protos.cmd @@ -0,0 +1,14 @@ +@setlocal + +@REM set HOME=%HOMEDRIVE%\%HOMEPATH% +@REM set REPO=%HOME +@REM for /f "usebackq tokens=*" %%a in (`echo Test`) do my_command %%a +set TOOLS_DIR=%~dp0 +set REPO_DIR=%~dp0..\ +set BUILD_SPARK_PY=%TOOLS_DIR%\py\build_spark.py +set PROTOC=%REPO_DIR%externals\windows\bin\protoc.exe +@REM c:\Users\John\Documents\repos\bts\spark\externals\windows\bin\protoc.exe C:\Users\John\Documents\repos\bts\spark\src\lib\config\protos\data_stream_config.proto --error_format=msvs --proto_path +@REM =c:\Users\John\Documents\repos\bts\spark\src --plugin="protoc-gen-spark-csharp=c:\Users\John\Documents\repos\bts\spark\src\lib\api\service_compiler\plugin.cmd" --spark-csharp_out=lang=c#,templates=immutable.pb.cs,strip_out_path=1:c:\Users\John\Documents\repos\bts\spark\build\Debug\obj\DataStrea +@REM mCore/lib/config/protos --csharp_out=c:\Users\John\Documents\repos\bts\spark\build\Debug\obj\DataStreamCore/lib/config/protos" + +%PROTOC% %REPO_DIR%src\lib\config\protos\data_stream_config.proto --error_format=msvs --proto_path=%REPO_DIR%src --plugin="protoc-gen-spark-csharp=%REPO_DIR%src\lib\api\service_compiler\plugin.cmd" --spark-csharp_out=lang=c#,templates=immutable.pb.cs,strip_out_path=1:%REPO_DIR%build\Debug\obj\DataStreamCore\lib\config\protos --csharp_out=%REPO_DIR%build\Debug\obj\DataStreamCore\lib\config\protos diff --git a/tools/pack_sparkd.sh b/tools/pack_sparkd.sh new file mode 100644 index 000000000..17d70b1e1 --- /dev/null +++ b/tools/pack_sparkd.sh @@ -0,0 +1,85 @@ +#!/usr/bin/env bash + +function _exit() { + if [[ "$#" -gt 1 ]]; then + # When there are multiple args, the last arg is the exit code. + echo "${@:1:($# - 1)}" + exit "${@: -1}" + fi + echo "$1" + exit 0 +} + +help_text="\ +Package a self-contained sparkd executable in a tar.gz file + +Usage: pack_sparkd.sh [OPTIONS] + +Positionals: + OUT The output path for the resultant sparkd tar.gz + +Options: + --out OUT The output path for the resultant sparkd tar.gz + --force Overwrite existing sparkd tar.gz + -h,--help Print this help message and exit + +" + +dest="" +pkg_name="" +declare -i overwrite_dest=0 + +while [[ $# -gt 0 ]]; do + case "$1" in + --out) + if [[ -z "$2" ]]; then + _exit "OUT (--out) must be non-empty" 1 + fi + dest="$2" + shift 2 + ;; + --force) + overwrite_dest=1 + shift 1 + ;; + -h) + ;& + --help) + _exit "$help_text" 0 + ;; + *) + esac +done + +tools_dir="$(dirname "$(readlink -f ${BASH_SOURCE[0]})")" +repo_dir="$tools_dir/.." +build_dir="$repo_dir/build/gcc/release" +install_dir="$repo_dir/install/Release/" +revision=$(git rev-parse --short HEAD) +timestamp=$(date +'%Y-%m%d-%H%M') + +if [[ -z "$dest" ]]; then + pkg_name="sparkd_${revision}_${timestamp}.tar.gz" + dest="$repo_dir/$pkg_name" +else + pkg_name="$(basename "$dest")" +fi + +if [ -e "$dest" ]; then + echo "The output file '$dest' already exists (use --force to overwrite)" + exit 1 +fi +eval "\ + $tools_dir/build_sparkd.sh --compiler gcc --build-type release && \ + cmake --build $build_dir --target install" +if [[ $? -ne 0 ]]; then + exit $? +fi +cd "$install_dir" +echo "Creating '$pkg_name'" +tar -czf "$pkg_name" "sparkd" +if [[ $? -ne 0 ]]; then + exit $? +fi + +mv "$pkg_name" "$dest" diff --git a/tools/py/gen_cmd_aliases.py b/tools/py/gen_cmd_aliases.py new file mode 100644 index 000000000..63891a9d7 --- /dev/null +++ b/tools/py/gen_cmd_aliases.py @@ -0,0 +1,79 @@ +import argparse +import os +from pathlib import Path +import subprocess + +from core.file_utils import SparkPaths, create_dir, find_files, find_file_tree +from core.iter_utils import at_bounded +from core.logging import exit_msg + + +def _get_arg_parser(): + parser = argparse.ArgumentParser( + description='Generates cmd scripts (aliases) for py scripts') + parser.add_argument( + '--sources', + nargs='+', + help='The source files for which to generate cs', + ) + parser.add_argument( + '--out-dir', + metavar='PATH', + help='The directory in which to dump the aliases', + ) + parser.add_argument( + '--python-exe', + metavar='PATH', + help='The python exe to use in the cmd aliases', + ) + parser.add_argument( + '--clean', + action='store_true', + help='Delete existing lexer.cs and parser.cs files in --out-dir', + ) + parser.add_argument( + '--overwrite-existing', + action='store_true', + help= + 'Overwrite existing files with the generated aliases (if necessary)', + ) + parser.add_argument( + '--recursive', + help='Whether or not to generate aliases recursively', + ) + this_dir = Path(__file__).parent.absolute() + parser.set_defaults( + sources=[this_dir], + python_exe='python', + clean=False, + ) + return parser + + +def _main(): + parser = _get_arg_parser() + args = parser.parse_args() + if args.sources is None: + print('No sources specified') + sources = [Path(s).absolute() for s in args.sources] + scripts = [f.path for s in sources for f in find_file_tree(s, '.py', 1)] + static_out_dir = None if args.out_dir is None else Path(args.out_dir) + + for script in scripts: + out_dir = static_out_dir or script.parent.with_name( + f'{script.parent.name}.aliases') + create_dir(out_dir) + alias = out_dir.joinpath(f'{script.stem}.cmd') + if alias.exists(): + print(f'Skipping {alias} because it already exists') + continue + print(f'Generating {alias}') + with open(alias, 'w') as alias_out: + alias_out.write(f'@{args.python_exe} {script.absolute()} %*\n') + + +if __name__ == "__main__": + try: + _main() + except Exception as e: + print(f'gen_cmd_aliases.py failed: {e}') diff --git a/tools/spark_dev_prompt.bat b/tools/spark_dev_prompt.bat new file mode 100644 index 000000000..300ac5e7b --- /dev/null +++ b/tools/spark_dev_prompt.bat @@ -0,0 +1 @@ +@"C:\\Program Files\\Microsoft Visual Studio\\2022\\Community\\Common7\\Tools\\VsDevCmd.bat" -arch=x64 -host_arch=x64 && %~dp0..\\.venv\\Scripts\\activate.bat