diff --git a/src/DataStreamSuite/DataStreamCore/Services/DataStreamCoreServices.cs b/src/DataStreamSuite/DataStreamCore/Services/DataStreamCoreServices.cs index 843ff8d51..2b376c499 100644 --- a/src/DataStreamSuite/DataStreamCore/Services/DataStreamCoreServices.cs +++ b/src/DataStreamSuite/DataStreamCore/Services/DataStreamCoreServices.cs @@ -31,5 +31,7 @@ public static class DataStreamCoreServices { .AddSingleton(sp => sp.GetRequiredService()); public static IServiceCollection AddCompositeDataStreamFactory(this IServiceCollection services) => - services.AddSingleton(); + services.AddSingleton() + // CompositeDataStreamFactory implements IHostedService to propagate itself to IRecursiveDataStreamFactory(s) + .AddHostedService(sp => sp.GetRequiredService()); } diff --git a/src/DataStreamSuite/SharedStreams/BlueOrb/Prompts/BlueOrbDSL_MLP_v1.9.md b/src/DataStreamSuite/SharedStreams/BlueOrb/Prompts/BlueOrbDSL_MLP_v1.9.md new file mode 100644 index 000000000..a482a8ca3 --- /dev/null +++ b/src/DataStreamSuite/SharedStreams/BlueOrb/Prompts/BlueOrbDSL_MLP_v1.9.md @@ -0,0 +1,1335 @@ +# BlueOrbDSL Master Learning Prompt v1.8 (Final) + +**This version is the *final, frozen* canonical Master Learning Prompt for BlueOrbDSL model training and generation.** **No prior versions apply. This document is the sole authoritative source.** +**No further edits are required. This is the single source of truth for Learning Mode.** +**Reviewed and validated as optimal on Aug 5th, 2025 after full Meta Mode review.** + +## 🔑 Core Rules and How to Use This DSL + +This document defines the **authoritative specification** for the BlueOrbDSL. All streams, operators, syntax, and behaviors must adhere strictly to what is explicitly documented below. + +### ✅ Allowed Behavior: + +- Use **only** the streams, operators, and functions explicitly listed in this document. +- Compose streams using the exact syntax, argument structure, and field names provided here. +- When in doubt, default to **conservative correctness over guessing**. + +### ❌ Forbidden Behavior: + +- **Do not invent** new streams, operators, behaviors, or field names that are not part of this document. +- **Do not assume** behavior based on other languages, prior versions, or intuitive expectations. +- If a requested DSL cannot be satisfied strictly within this specification, **do not generate it**—instead, clearly respond that the instruction is outside the permitted scope. + +### 📌 Mandatory Validation Checklist (Before Every DSL Generation): + +1. Is **every stream** in the generated DSL explicitly defined in this document? +2. Is **every operator** used exactly as specified here—no missing, renamed, or invented components? +3. Is the **syntax strictly consistent** with the EBNF rules provided? + +If any answer is “No,” stop and return: + +⚠️ "This instruction cannot be completed within the current BlueOrbDSL specification." + +### 💡 Design Philosophy: + +This DSL is designed for: + +- Determinism +- Strict reproducibility +- Zero tolerance for unauthorized extensions + +Any helpfulness that introduces “close but wrong” behavior is considered a critical failure. + +--- + +# Table of Contents + +1. Preamble +2. Streams Overview +3. Data Sources + 3.1 FutureHistoricalBarsStream + 3.2 LiveBBOStream +4. Operators + 4.1 FilterStream + 4.2 ProjectStream + 4.3 GroupByStream + 4.4 SingleRowMathStream + 4.5 MultiRowRollingMathStream + 4.6 UnionStream + 4.7 JoinStream + 4.8 DateTimeExpanderStream + 4.9 ContractExpanderStream + 4.10 CountStream + 4.11 TimeSeriesStream + 4.12 TimeBasedRollingFilterStream + 4.13 PythonDataframeStream +5. Reserved Words & Enumerations +6. Common Pitfalls + +--- + +## 1 Preamble + +BlueOrbDSL turns natural-language finance questions into **stream pipelines** that operate on tabular market data (historical or real-time). +Each *stream* is either a **Data Source** (fetches data) or an **Operator** (transforms data). +Streams compose: the output table of one becomes the input of the next. + +**Convention:** For all operators, the **first argument** is always the `parent_stream`. Subsequent arguments are constructor parameters. + +--- + +## 2 Streams Overview + +All stream definitions follow this table layout: + +| Field | Meaning | +| :---- | :---- | +| **Type** | DataSource / Operator | +| **Purpose** | One-sentence description | +| **Arguments** | Bullet list; first arg \= parent\_stream (if Operator) | +| **Output Schema** | Columns created or removed | +| **Adds Columns?** | Yes / No | +| **Modifies Schema?** | Yes / No | +| **Example Usage** | Fenced `dsl` block | +| **Special Notes** | Constraints / caveats | + +--- + +## 3 Data Sources + +### 3.1 FutureHistoricalBarsStream + +| Field | Description | +| :---- | :---- | +| **Type** | DataSource | +| **Purpose** | Fetch historical OHLCV bars for futures contracts | +| **Arguments** | • `contract` — continuous (`ES`), specific (`ESZ9`), or wildcard (`ES*`) • \``bar_size`\` — ONE\_DAY, ONE\_MIN • `` `range_begin` `` — ISO datetime, optional • `` `range_end` `` — ISO datetime, optional | +| **Output Schema** | `id, ts_event, date, contract, open, high, low, close, volume` | +| **Adds Columns?** | Yes | +| **Modifies Schema?** | No | +| **Example Usage** | `bars = FutureHistoricalBarsStream("ES", "ONE_DAY", "2023-01-01", "2023-12-31")` | +| **EBNF** | See [Appendix Z](#appendix-z--formal-grammar-ebnf), rule `FutureHistoricalBarsStream` | +| **Special Notes** | Whereas the ts\_event column is the timestamp of the column, date is the date of the day. | + +### 3.2 LiveBBOStream + +| Field | Description | +| :---- | :---- | +| **Type** | DataSource | +| **Purpose** | Stream live best-bid/offer quotes | +| **Arguments** | • `contract` — e.g. `"NGU5"` or `"CL*"` | +| **Output Schema** | `ts_event, contract, bid_price, bid_size, ask_price, ask_size` | +| **Adds Columns?** | Yes | +| **Modifies Schema?** | No | +| **Example Usage** | `bbo = LiveBBOStream("NGU5")` | +| **EBNF** | See [Appendix Z](#appendix-z--formal-grammar-ebnf), rule `FutureHistoricalBarsStream` | +| **Special Notes** | LiveBBOStream produces the latest value of the bid/ask price and size. It does not maintain history. Each new row overwrites the old one. For history use TimeSeriesStream | + +### 3.3 LiveTradeStream + +| Field | Description | +| :---- | :---- | +| **Type** | DataSource | +| **Purpose** | Stream live time series of trades | +| **Arguments** | • `contract` — e.g. `"NGU5"` or `"CL*"` | +| **Output Schema** | `ts_event, contract, trade_size, trade_price,aggressor_side(BUY or SELL),volume` | +| **Adds Columns?** | Yes | +| **Modifies Schema?** | No | +| **Example Usage** | `bbo = LiveTradeStream("NGU5")` | +| **EBNF** | See [Appendix Z](#appendix-z--formal-grammar-ebnf), rule `LiveTradeStream` | +| **Special Notes** | LiveTradeStream produces a circular buffer of all trades at the exch Do not wrap LiveTradeStream in TimeSeriesStream — it already provides a circular buffer of time series data. • Does not include OHLCV columns (e.g., open/close); Only use documented output schema. | + +--- + +## 4 Operators + +### 4.1 FilterStream + +| Field | Description | +| :---- | :---- | +| **Type** | Operator | +| **Purpose** | Keep rows where a boolean condition is true | +| **Arguments** | • `parent_stream` •`` `condition` `` — string | +| **Output Schema** | Same as input | +| **Adds Columns?** | No | +| **Modifies Schema?** | No | +| **Example Usage** | | +| **EBNF** | See [Appendix Z](#appendix-z--formal-grammar-ebnf), rule `FilterStream` | +| **Special Notes** | Condition syntax: use ASCII single quotes (`contract = 'ESZ9'`). Supports `and`, `or`, parentheses, equality and inequality. Does not support ‘IN’ Conditions must reference existing columns only. Function calls like \`abs(x)\` or inline math expressions are not allowed inside the condition string. To filter on a derived value (e.g., \`abs(z\_score) \> 4\`), first compute \`abs\_z \= abs(z\_score)\` using \`SingleRowMathStream\`. | + +### 4.2 ProjectStream + +| Field | Description | +| :---- | :---- | +| **Type** | Operator | +| **Purpose** | Rename or drop columns | +| **Arguments** | • `parent_stream` • `` `column_map ``\` — string | +| **Output Schema** | Renamed subset | +| **Adds Columns?** | No | +| **Modifies Schema?** | Yes | +| **Example Usage** | `clean = ProjectStream(bars, "close:settlement, volume")` | +| **EBNF** | See [Appendix Z](#appendix-z--formal-grammar-ebnf), rule `ProjectStream` | +| **Special Notes** | ProjectStream may only select and optionally rename existing columns. It **cannot** add new constant or computed columns. Use `AddColumnStream` for that purpose. | + +--- + +### + +### + +### 4.3 GroupByStream + +| Field | Description | +| :---- | :---- | +| **Type** | Operator | +| **Purpose** | Aggregates rows by key (or globally) | +| **Arguments** | • `parent_stream` • `` `group_by` `` — string (blank for global) • `` `aggregation_string` `` — string | +| **Output Schema** | Adds `_` columns | +| **Adds Columns?** | Yes | +| **Modifies Schema?** | Yes | +| **Example Usage** | `daily = GroupByStream(bars, "date", "volume:Sum")` | +| **EBNF** | See [Appendix Z](#appendix-z--formal-grammar-ebnf), rule `GroupByStream` | +| **Special Notes** | Aggregations can **only** reference existing column names, not expressions. To aggregate an expression (e.g., `close * volume`), first define it with `SingleRowMathStream`. **Special Notes** GroupByStream supports time-based grouping via fields like `ts_event_date`, `ts_event_hour`, etc. It supports global aggregation when `group_by = ""` Supports multiple aggregations using a comma-separated string GroupByStream **preserves the input schema**: all existing columns from the parent stream remain unchanged. It **adds new columns** based on the aggregation expressions provided (e.g., `volume_sum`, `close_max`, etc.). This means the output schema is an **extension**, not a replacement — unlike in SQL-style group-by operations. Any column present in both the input and the aggregation result will trigger prefixing in subsequent JoinStream operations. If the output of GroupByStream is part of a final user-facing result (e.g., a single-row mean), ensure the pipeline ends with a \`ProjectStream\` to emit only the relevant column(s), with clean naming. | + +### 4.4 SingleRowMathStream + +| Field | Description | +| :---- | :---- | +| **Type** | Operator | +| **Purpose** | Add new columns from row-wise math expressions | +| **Arguments** | • `parent_stream` • `` `math_expression` `` — string | +| **Output Schema** | Adds new column(s) | +| **Adds Columns?** | Yes | +| **Modifies Schema?** | No | +| **Example Usage** | `math = SingleRowMathStream(bars, "log_return = Log(close / open)")` | +| **EBNF** | See [Appendix Z](#appendix-z--formal-grammar-ebnf), rule `SingleRowMathStream` | +| **Special Notes** | **Special Notes** | Operates strictly on a **single row** Only supports **arithmetic expressions** using existing numeric columns Math functions like `Log`, `Abs`, `Sin`, `Cos`, etc. must use **capitalized names**, matching the casing of C\#'s `System.Math` API. For example, write `Log(x)` not `log(x)`. ❌ **Boolean logic, comparisons, and conditionals are not allowed** (e.g., `trigger = x > 1.0` is invalid) ✅ To filter on conditions, use `FilterStream("x > 1.0")` instead For cross-row logic, use `MultiRowRollingMathStream` (e.g., `close:EdgeDiff`) or `PythonDataframeStream` | + +### 4.5 MultiRowRollingMathStream + +| Field | Description | +| :---- | :---- | +| **Type** | Operator | +| **Purpose** | Aggregates over rolling window of rows | +| **Arguments** | • `parent_stream` • `` `aggregation_string` `` — string • `` `sort_column` `` — string with \+/- prefix • `` `rolling_window_size` `` — int | +| **Output Schema** | Adds `_` columns | +| **Adds Columns?** | Yes | +| **Modifies Schema?** | Yes | +| **Example Usage** | `ma20 = MultiRowRollingMathStream(bars, "close:Mean", "-ts_event", 20)` | +| **EBNF** | See [Appendix Z](#appendix-z--formal-grammar-ebnf), rule `MultiRowRollingMathStream` | +| **Special Notes** | sort\_column should be prefixed with \+ (ascending) or \- (descending). Example: "-ts\_event". **Note:** This stream operates on a fixed *row count*. For time-based windows (e.g. last 30 seconds), use `TimeBasedRollingFilterStream` \+ `GroupByStream`. This operator **retains all input columns by default** and **adds new rolling aggregation columns**. For example, applying `close:Mean` will return all columns from the parent stream, plus a new `close_mean` column. | + +### 4.6 UnionStream + +| Field | Description | +| :---- | :---- | +| **Type** | Operator | +| **Purpose** | Append rows from two streams | +| **Arguments** | • `parent_stream1` • `parent_stream2` | +| **Output Schema** | Same as inputs | +| **Adds Columns?** | No | +| **Modifies Schema?** | No | +| **Example Usage** | `unioned = UnionStream(bars1, bars2)` | +| **EBNF** | See [Appendix Z](#appendix-z--formal-grammar-ebnf), rule `MultiRowRollingMathStream` | +| **Special Notes** | **Schemas must match exactly (column names AND types).** | + +### 4.7 JoinStream + +| Field | Description | +| :---- | :---- | +| **Type** | Operator | +| **Purpose** | Join two streams on equality of selected columns | +| **Arguments** | • `` `left_parent_stream` `` • `` `left_join_cols` `` — comma-separated string • `` `right_parent_stream` `` • `` `right_join_cols` `` — comma-separated string | +| **Output Schema** | Combined columns from both streams | +| **Adds Columns?** | Yes | +| **Modifies Schema?** | Yes | +| **Example Usage** | **Example Usage** `joined = JoinStream(a, "ts_event_date", b, "ts_event_date")` | +| **EBNF** | See [Appendix Z](#appendix-z--formal-grammar-ebnf), rule `JoinStream` | +| **Column Naming Behavior** | Column Naming Behavior The output schema of `JoinStream` is determined by the following rule: Any column name that appears in **both parent streams** — regardless of whether it is a join key — will be prefixed with `left_` and `right_`. Columns that appear in **only one parent** are retained as-is, with no prefix. This rule ensures deterministic, non-colliding column names across all join configurations. **Case Column Appears In Output Name** A Both left & right parents `left_`, `right_` B Only one parent (left or right) `` 💡 *Use `ProjectStream` after a join to rename or clean up prefixed columns as needed.* | +| **Special Notes** | Always performs an **inner join** on equality. Any column name present in **both** parent streams will be renamed with `left_` and `right_` prefixes — **even if it is a join key**. Columns that appear in **only one stream** retain their original names. This ensures deterministic output schemas and avoids ambiguity. If join columns args left empty, does cartesian product When joining two `LiveBBOStream` outputs (e.g., `ES` and `NQ`), joining on `ts_event` will often fail since each emits only the latest tick with distinct timestamps. \+Use empty join keys (`""`) to form a **cartesian product**, pairing the latest bid/ask data from each contract — ideal for synthetic instruments or spread monitoring. | + +### 4.8 DateTimeExpanderStream + +| Field | Description | +| :---- | :---- | +| **Type** | Operator | +| **Purpose** | Adds columns with date/time components from a datetime column | +| **Arguments** | • `parent_stream` • `datetime_column` — string | +| **Output Schema** | Adds: • \`\_date\` • \`\_day\_of\_week\` • \`\_day\_of\_month\` • \`\_month\` • \`\_hour\` • \`\_minute\` • \`\_seconds\_of\_day\` | +| **Adds Columns?** | Yes | +| **Modifies Schema?** | Yes | +| **Example Usage** | `expanded = DateTimeExpanderStream(bars, "ts_event")` | +| **EBNF** | See [Appendix Z](#appendix-z--formal-grammar-ebnf), rule `DateTimeExpanderStream` | +| **Special Notes** | Aggregations can **only** reference existing column names, not expressions. To aggregate an expression (e.g., `close * volume`), first define it with `SingleRowMathStream`. | + +### 4.9 ContractExpanderStream + +| Field | Description | +| :---- | :---- | +| **Type** | Operator | +| **Purpose** | Adds metadata columns based on `contract` | +| **Arguments** | • `parent_stream` • contract\_column — string | +| **Output Schema** | Adds: `• contract_type` | +| **Adds Columns?** | Yes | +| **Modifies Schema?** | Yes | +| **Example Usage** | `expanded = ContractExpanderStream(bars, "contract")` | +| **EBNF** | See [Appendix Z](#appendix-z--formal-grammar-ebnf), rule `ContractExpanderStream` | +| **Special Notes** | Aggregations can **only** reference existing column names, not expressions. To aggregate an expression (e.g., `close * volume`), first define it with `SingleRowMathStream`. | + +### 4.10 CountStream + +| Field | Description | +| :---- | :---- | +| **Type** | Operator | +| **Purpose** | Count number of rows | +| **Arguments** | • `parent_stream` | +| **Output Schema** | `id, count` | +| **Adds Columns?** | Yes | +| **Modifies Schema?** | Yes | +| **Example Usage** | `total_count = CountStream(bars)` | +| **EBNF** | See [Appendix Z](#appendix-z--formal-grammar-ebnf), rule `CountStream` | +| **Special Notes** | Aggregations can **only** reference existing column names, not expressions. To aggregate an expression (e.g., `close * volume`), first define it with `SingleRowMathStream`. | + +### 4.11 TimeSeriesStream + +| Field | Description | +| :---- | :---- | +| **Type** | Operator | +| **Purpose** | Append each update from parent stream into time series | +| **Arguments** | • `parent_stream` • \``buffer_size`\` — int | +| **Output Schema** | Parent schema \+ `timestamp` (if not already present) | +| **Adds Columns?** | Yes | +| **Modifies Schema?** | No | +| **Example Usage** | `ts_bbo = TimeSeriesStream(bbo, 10000)` | +| **EBNF** | See [Appendix Z](#appendix-z--formal-grammar-ebnf), rule `TimeSeriesStream` | +| **Special Notes** | Append-only stream, does not deduplicate or collapse repeated values | + +### 4.12 TimeBasedRollingFilterStream + +| Field | Description | +| :---- | :---- | +| **Type** | Operator | +| **Purpose** | Keep rows newer than rolling window of N seconds | +| **Arguments** | • `parent_stream` • `` `time_column` `` — string • \``no_of_seconds`\` — int | +| **Output Schema** | Same as input | +| **Adds Columns?** | No | +| **Modifies Schema?** | No | +| **Example Usage** | `recent = TimeBasedRollingFilterStream(trades, “ts_event”, 60)` | +| **EBNF** | See [Appendix Z](#appendix-z--formal-grammar-ebnf), rule `TimeBasedRollingFilterStream` | +| **Special Notes** | Drops rows older than `no_of_seconds` compared to current max timestamp in the \`time\_column\` Requires an explicit time column to anchor the comparison | + +### 4.13 PythonDataframeStream + +| Field | Description | +| :---- | :---- | +| **Type** | Operator | +| **Purpose** | Apply arbitrary Python transformation to DataFrame | +| **Arguments** | • `parent_stream` • `` `python_function` `` — string | +| **Output Schema** | Defined by function | +| **Adds Columns?** | Yes/No — depends on function | +| **Modifies Schema?** | Yes/No — depends on function | +| **Example Usage** | `r = PythonDataframeStream(bars, "def transform(df): return df[df.volume > 1e6]")` | +| **EBNF** | See [Appendix Z](#appendix-z--formal-grammar-ebnf), rule `PythonDataframeStream` | +| **Special Notes** | **Use PythonDataframeStream when and only when**—the calculation is fundamentally “stateful,” cumulative, or requires logic that’s impossible in the canonical DSL. **Use only as a last resort.** Function should be small and well-documented. Avoid large multi-step logic where possible. The returned DataFrame **must always include a well-defined schema**, even when no rows are emitted. If the computed result is empty, you should still return an empty `pd.DataFrame` **with explicit column names**, using the `columns=` argument. This ensures compatibility with downstream streams like `ProjectStream`, which will error if they receive a schema-less `{}`. ✅ Recommended pattern: `return pd.DataFrame(rows, columns=["col1", "col2", ...])` | + +### 4.14 TopStream + +| Field | Description | +| :---- | :---- | +| **Type** | Operator | +| **Purpose** | To select the top N rows of the parent\_stream, sorted by given columns, ascending or descending | +| **Arguments** | • `parent_stream` • `number of rows` — int • `sort columns` — string | +| **Output Schema** | Same as parent | +| **Adds Columns?** | No | +| **Modifies Schema?** | No — same as parent stream | +| **Example Usage** | `r = TopStream(bars, 10, “-volume, +contract”)` | +| **EBNF** | See [Appendix Z](#appendix-z--formal-grammar-ebnf), rule `TopStream` | +| **Special Notes** | Use \- for descending order, \+ for ascending | + +--- + +## 5 Reserved Words & Enumerations + +- **Bar\_size:** `ONE_DAY`, `ONE_MIN` +- **Aggregation Operations:** `Mean`, `Median`, `Mode`, `Sum`, `Max`, `Min`, `StdDev`, `Range`, `First`, `Last`, `EdgeDiff`, `EdgeDiffPercentage` + +### 5.1 Contract Naming Conventions + +- Use **continuous contract codes** (e.g., `"ES"`, `"CL"`, `"ZN"`) when the natural language query refers to: + - “front-month” + - “active contract” + - “generic futures” + - “rolling series” +- These continuous symbols represent **auto-rolling front-month contracts**, consistent with typical market convention and backtesting best practices. +- Use specific codes (e.g., `"ESU5"`) only when the question explicitly references a month/year or fixed expiry. +- Use wildcard (e.g., `"ES*")` when the question references all ES contracts, futures or spreads. + +--- + +## 6 Common Pitfalls + +| Pitfall | Correct Approach | +| :---- | :---- | +| Using expressions like `close*volume:Sum` in `GroupByStream` | Use `SingleRowMathStream` to define `dollar_value = close * volume`, then aggregate: `dollar_value:Sum` | +| Smart quotes in conditions | Use `'...'` ASCII quotes | +| Large functions in PythonDataframeStream | Keep minimal or move to appendix | +| Ambiguous TimeBasedRollingFilterStream behavior | Now clearly stated: drops rows older than N seconds. | +| Using `abs(...)` or other functions in `FilterStream, GroupByStream, MultiRowRollingMathStream` conditions | First compute derived column with `SingleRowMathStream`, then use it | +| Using \`abs(...)\` or other functions in `FilterStream, GroupByStream, MultiRowRollingMathStream` conditions | First compute derived column with `SingleRowMathStream`, then filter on that | +| Using `MultiRowRollingMathStream` to compute moving average over time (e.g., 30 seconds) | Use `TimeSeriesStream` → `TimeBasedRollingFilterStream` → `GroupByStream` instead. `MultiRowRollingMathStream` uses fixed row counts, not durations. | +| Joining two LiveBBOStream outputs using `ts_event` | Use `JoinStream(a, "", b, "")` to perform a cartesian product. `LiveBBOStream` emits only one row per stream with potentially non-matching timestamps. Joining on `ts_event` is brittle and may result in empty joins. | +| Using \`close\[-1\]\` or \`open\[-2\]\` in \`SingleRowMathStream\` | Use \`MultiRowRollingMathStream(..., "close:EdgeDiff", ...)\` or \`PythonDataframeStream\` for multi-row logic | +| Using `LiveTradeStream` as if it had OHLC columns like `open` or `close` | Use only `trade_price`, `trade_size`, etc. Compute returns from adjacent trade prices in `PythonDataframeStream` | +| Wrapping `LiveTradeStream` in `TimeSeriesStream` | Not needed — it's already a time series with historical buffer | +| Joining on `ts_event` when timestamps may not match exactly | Avoid joining on `ts_event` unless you're certain both sides align at the exact timestamp. This is rarely safe outside controlled bar-aligned streams. Prefer joining on coarser keys like `ts_event_date`, or use `DateTimeExpanderStream` to derive appropriate join fields. | +| Assuming join keys are merged into one column | All columns shared across both inputs are prefixed, **including join keys**. Use `ProjectStream` after JoinStream to select and rename. | +| Trying to create a boolean column like `trigger = x > 1.0` in `SingleRowMathStream` | Use `FilterStream("x > 1.0")` instead. `SingleRowMathStream` supports only arithmetic expressions. | +| ❗ **Assuming that GroupByStream drops existing columns**: | Unlike SQL-style group-bys, this operator **preserves all original columns** and simply adds the aggregation results. This can lead to unintended prefixing in downstream JoinStream operations if you don’t explicitly project what you need. | +| ❗ **Returning all columns by default**:. | If a pipeline ends in a computed or aggregated value, failing to use ProjectStream can result in verbose or irrelevant outputs (e.g., schema clutter with unused base columns like ts\_event, volume, etc.). Use ProjectStream at the end to shape the final result. | + +## + +## 7 Reinforced Learning DSL Examples + +### 7.1 Demo-Informed Canonical Examples + +## The following queries (in section 7.1) are **annotated as Demo-Informed**. They serve as **hard anchors** for DSL generation. When a natural language query is **identical or structurally similar**, the model should copy or adapt the DSL **directly from these examples**, instead of generating new logic. + +## Similarity includes: + +- ## Same structure, but different contract (e.g. `CL` vs `NG`) + +- ## Different threshold (e.g. `> 2.5%` vs `> 3.0%`) + +- ## Paraphrased language (e.g. “gap down” vs “opens lower by”) + +## + +## These are **trusted, production-grade canonical mappings**. + +##### Example D1 — \[Demo-Informed Query\] NFP Event vs Historical High-Low Range + +📌 *This example is canonical. If a future query involves economic events, time windows, and comparative volatility or range analysis — especially using unioned event slices — replicate this scaffold.* + +**NL Query:** +What was the high-low range in 10-year Treasury futures during the 15 minutes after each of the NFP announcements in Q3 and Q4 2024, and how does it compare to the average 10-minute high-low range during the same period? + +(Shows integration of economic events, precise time slicing, and aggregation over broader history.) +DSL: + +``` + +// Load 15-minute windows after each known NFP release (12:30–12:44:59 UTC) +// These are treated as fixed event windows, not computed via filtering +nfp1 = FutureHistoricalBarsStream("ZN", "ONE_MIN", "2024-07-05T12:30:00", "2024-07-05T12:44:59") +nfp2 = FutureHistoricalBarsStream("ZN", "ONE_MIN", "2024-08-02T12:30:00", "2024-08-02T12:44:59") +nfp3 = FutureHistoricalBarsStream("ZN", "ONE_MIN", "2024-09-06T12:30:00", "2024-09-06T12:44:59") +nfp4 = FutureHistoricalBarsStream("ZN", "ONE_MIN", "2024-10-04T12:30:00", "2024-10-04T12:44:59") +nfp5 = FutureHistoricalBarsStream("ZN", "ONE_MIN", "2024-11-01T12:30:00", "2024-11-01T12:44:59") +nfp6 = FutureHistoricalBarsStream("ZN", "ONE_MIN", "2024-12-06T12:30:00", "2024-12-06T12:44:59") + +// Combine Q3 and Q4 event windows using nested UnionStreams +// This pattern avoids using filters or expressions to extract event periods +q3 = UnionStream(UnionStream(nfp1, nfp2), nfp3) +q4 = UnionStream(UnionStream(nfp4, nfp5), nfp6) +nfp_bars = UnionStream(q3, q4) + +// For each NFP event, compute high/low range from its 15-minute window +nfp_ranges = GroupByStream(nfp_bars, "date", "high:Max, low:Min") +nfp_ranges = SingleRowMathStream(nfp_ranges, "range = high_max - low_min") + +// Load all 1-minute bars in Q3 + Q4 for comparison baseline +all_bars = FutureHistoricalBarsStream("ZN", "ONE_MIN", "2024-07-01T00:00:00", "2024-12-31T23:59:59") + +// Use Python to compute rolling 10-minute high-low ranges across the full period +// This is required because MultiRowRollingMathStream can't aggregate high/low across multiple rows +all_rolling_ranges = PythonDataframeStream(all_bars, " +def transform(df): + import pandas as pd + out = [] + df = df.sort_values('ts_event') + for i in range(len(df) - 9): + window = df.iloc[i:i+10] + rng = window['high'].max() - window['low'].min() + out.append({'window_start': window['ts_event'].iloc[0], 'range_mean': rng}) + return pd.DataFrame(out) +") + +// Compute the overall average of all 10-minute high-low ranges across the period +avg_range = GroupByStream(all_rolling_ranges, "", "range_mean:Mean") + +// Perform a cartesian join to attach average context to each event (no keys) +// This avoids mismatched timestamps and ensures every NFP row gets context +final = JoinStream(nfp_ranges, "", avg_range, "") + +// Output event-specific and reference stats, with renamed columns for clarity +result = ProjectStream(final, " + date, + high_max:event_high, + low_min:event_low, + range:event_high_low_range_15min, + range_mean:avg_10min_high_low_range_q3_q4 +") +``` + +### Example D2 — \[Demo-Informed Query\] SMA-based Entry and Exit with Trade Tracking and Slippage + +📌 *This example is canonical. If a future query involves backtesting entry/exit based on rolling conditions, especially with slippage, position state, and end-of-period forcing, this scaffold should be directly reused or adapted.* + +**NL Query:** +Show me every trade I would have made in Q2 of 2024 if I had gone long Nasdaq futures whenever the market opened below its 1-week simple moving average. Close the position as soon as that condition no longer holds, or force an exit on the final trading day of the quarter. For each trade, display the entry and exit as separate rows, the continuous contract, include the reason for the action, and calculate both the gross and net profit or loss, accounting for 0.1% slippage on both entry and exit. Also track the cumulative net PnL and position over time. + +``` +// Load daily bars for Nasdaq futures for Q2 2024 +bars = FutureHistoricalBarsStream("NQ", "ONE_DAY", "2024-04-01T00:00:00", "2024-06-30T23:59:59") + +// Compute 5-day simple moving average of the close price, sorted by ts_event +sma_5 = MultiRowRollingMathStream(bars, "close:Mean", "+ts_event", 5) + +// Join the original bars with the SMA values on timestamp +joined = JoinStream(bars, "ts_event", sma_5, "ts_event") + +// Select and rename columns for clarity before passing to Python +clean = ProjectStream(joined, " + left_ts_event:ts_event, + left_contract:contract, + left_open:open, + left_close:close, + close_mean:close_sma_5 +") + +// Use Python to model the strategy: +// - Enter long when open < 5-day SMA +// - Exit when that condition fails or on the last trading day +// - Track gross/net PnL, slippage, position state, and cumulative PnL +trades = PythonDataframeStream(clean, " +def transform(df): + import pandas as pd + df = df.sort_values('ts_event').reset_index(drop=True) + + records = [] + position_open = False + cumulative_net = 0.0 + entry_price = entry_slip = None + + for i, row in df.iterrows(): + ts = row.get('ts_event') + contract = row.get('contract') + open_px = row.get('open') + close_px = row.get('close') + sma = row.get('close_sma_5') + + # Skip rows with missing data + if pd.isnull([ts, contract, open_px, close_px, sma]).any(): + continue + + entry_condition = open_px < sma + is_last = i == len(df) - 1 + + # Entry logic: open < SMA, only if not already in position + if not position_open and entry_condition: + entry_price = open_px + entry_slip = entry_price * 1.001 # 0.1% slippage on entry + records.append({ + 'ts_event': ts, + 'event_type': 'entry', + 'contract': contract, + 'price': entry_price, + 'reason': 'open < 1-week SMA', + 'pnl_gross': None, + 'pnl_net': None, + 'cumulative_pnl_net': cumulative_net, + 'position': 1 + }) + position_open = True + + # Exit logic: condition fails or it's the last day + elif position_open and (not entry_condition or is_last): + exit_price = close_px + exit_slip = exit_price * 0.999 # 0.1% slippage on exit + pnl_gross = exit_price - entry_price + pnl_net = exit_slip - entry_slip + cumulative_net += pnl_net + records.append({ + 'ts_event': ts, + 'event_type': 'exit', + 'contract': contract, + 'price': exit_price, + 'reason': 'forced quarter-end exit' if is_last else 'exit: condition no longer holds', + 'pnl_gross': pnl_gross, + 'pnl_net': pnl_net, + 'cumulative_pnl_net': cumulative_net, + 'position': 0 + }) + position_open = False + entry_price = entry_slip = None + + return pd.DataFrame(records) +") +``` + +### Example D3 — \[Demo-Informed Query\] Streaming Volume Pressure from Buy Trades + +📌 *This example is canonical. When a query involves short-vs-long streaming window comparisons (e.g., buy volume over last 10 seconds vs. average over 2 minutes), this scaffold should be reused. Also serves as the correct usage pattern for TimeBasedRollingFilterStream → GroupByStream → JoinStream → Math.* + +**NL Query:** +Stream a real-time signal showing the intensity of buy-side pressure in E-mini S\&P futures by dividing total buy volume in the past 10 seconds by the average buy trade size over the past 2 minutes. Output the current `volume_pressure` as a streaming signal. + +``` +// Load live trades for E-mini S&P futures +trades = LiveTradeStream("ES") + +// Retain only buy-initiated trades based on aggressor side +buy_trades = FilterStream(trades, "aggressor_side = 'BUY'") + +// Retain buy trades from the last 10 seconds (short window) +short_window = TimeBasedRollingFilterStream(buy_trades, "ts_event", 10) + +// Retain buy trades from the last 2 minutes (long window) +long_window = TimeBasedRollingFilterStream(buy_trades, "ts_event", 120) + +// Compute total buy volume in the short window (sum of trade sizes) +short_vol = GroupByStream(short_window, "", "trade_size:Sum") + +// Compute average buy trade size in the long window +long_avg = GroupByStream(long_window, "", "trade_size:Mean") + +// Cartesian join (no keys) to combine both aggregates in one row +joined = JoinStream(short_vol, "", long_avg, "") + +// Compute volume pressure = total volume in short window / mean trade size in long window +signal = SingleRowMathStream(joined, " + volume_pressure = trade_size_sum / trade_size_mean +") + +// Output final signal as streaming value +result = ProjectStream(signal, " + volume_pressure +") +``` + +### Example D4 — \[Demo-Informed Query\] Real-Time Bollinger Squeeze Detection in Nasdaq Futures + +### 📌 *This example is canonical. When a query involves short-term volatility compression detection (e.g., rolling std dev relative to recent max), this scaffold should be reused. It teaches correct use of MultiRowRollingMathStream for tick windows, TimeBasedRollingFilterStream for normalization, and real-time streaming logic.* + +### + +### **NL Query:** Track Bollinger Band squeeze conditions in real time for Nasdaq futures. Compute the 20-tick rolling standard deviation of midprice and normalize it by its own 1-minute max. Stream the compression ratio continuously to detect setups where volatility contracts before a breakout. + +### + +``` +// Stream live best-bid-offer quotes for Nasdaq futures +bbo = LiveBBOStream("NQ") +// Compute midprice as average of bid and ask +mid = SingleRowMathStream(bbo, "mid = 0.5 * (bid_price + ask_price)") +// Convert midprice stream into a rolling time series buffer (tick history) +ts = TimeSeriesStream(mid, 10000) +// Compute rolling 20-tick standard deviation of midprice +rolling_std = MultiRowRollingMathStream(ts, "mid:StdDev", "-ts_event", 20) +// Extract only the last 60 seconds of standard deviation values +windowed = TimeBasedRollingFilterStream(rolling_std, "ts_event", 60) +// Compute the max stddev in the last 60 seconds +max_std = GroupByStream(windowed, "", "mid_stddev:Max") +// Cartesian join: combine each stddev with latest rolling max +joined = JoinStream(rolling_std, "", max_std, "") +// Normalize: compression ratio = current stddev / max stddev +squeeze = SingleRowMathStream(joined, " + compression_ratio = left_mid_stddev / mid_stddev_max +") +// Output timestamp and real-time compression signal +result = ProjectStream(squeeze, "left_ts_event:ts_event, compression_ratio") +// Return the latest 20 points (ordered descending by timestamp) +result = TopStream(result, 20, "-ts_event") +``` + +### + +### 7.2 General Learning Examples + +### The following examples are designed to provide **broad DSL coverage** across contracts, timeframes, use cases, and operators. + +### Unlike Demo-Informed Canonical Examples, these are used to: + +- ### Teach idiomatic stream composition + +- ### Reinforce correct usage of operators + +- ### Guide stylistic and naming conventions + +### + +### They serve as **training patterns**, not **strict templates**. When encountering similar queries, the model should **draw on these for inspiration**, but may generate new pipelines when appropriate + +### --- + +### Example G1 — Average Return and Volatility During First 10 Trading Days of Each Month + +**NL Query:** +What are the average return and volatility of CL during the first 10 trading days of each calendar month? + +``` +// Load daily bars for Crude Oil futures from 2010–2025 +bars = FutureHistoricalBarsStream("CL", "ONE_DAY", "2010-01-01T00:00:00", "2025-12-31T23:59:59") + +// Extract calendar components like month, day of week, etc. +expanded = DateTimeExpanderStream(bars, "ts_event") + +// Compute percent return from open to close +returns = SingleRowMathStream(expanded, "daily_return = 100 * (close - open) / open") + +// Filter to keep only the first 10 calendar trading days of each month +first_10 = FilterStream(returns, "ts_event_day_of_month <= 10") + +// Group by month and compute mean and standard deviation of daily return +aggregated = GroupByStream(first_10, "ts_event_month", "daily_return:Mean, daily_return:StdDev") + +// Rename columns for readability +result = ProjectStream(aggregated, "ts_event_month:month, daily_return_mean:avg_return, daily_return_stddev:volatility") + +// Optional: sort months from January (1) to December (12) +result = SortStream(result, "month") +``` + +--- + +### Example G2 — Detecting 2% Gap-Ups in SP500 Futures + +**NL Query:** +On which dates did SP 500 Futures open 2% higher than the previous day's close? + +``` +// Load daily bars for S&P 500 futures with an extended date range +bars = FutureHistoricalBarsStream("ES", "ONE_DAY", "1900-01-01T00:00:00", "2100-12-31T23:59:59") + +// For each 2-day window, extract: +// - previous day's close (as first row) +// - current day's open (as last row) +rolling_bars = MultiRowRollingMathStream(bars, "close:First, open:Last", "+ts_event", 2) + +// Compute percent change from previous close to current open +percent_change = SingleRowMathStream(rolling_bars, "percent_change = 100 * (open_last - close_first) / close_first") + +// Keep rows where the gap-up is at least 2% +filtered_bars = FilterStream(percent_change, "percent_change >= 2") + +// Return timestamp and percent change +result = ProjectStream(filtered_bars, "ts_event, percent_change") +``` + +--- + +### Example G3 — SP500 Q4 2023 Daily Returns as % of Q4 2022 Average Return + +**NL Query:** +Daily returns for SP500 futures, expressed as a percentage of the average SP return of Q4 2022\. + +``` +// Load daily bars for SP500 futures covering Q4 2022 and Q4 2023 +bars = FutureHistoricalBarsStream("ES", "ONE_DAY", "2022-10-01T00:00:00", "2023-12-31T23:59:59") + +// Compute daily return as percent change from open to close +returns = SingleRowMathStream(bars, "daily_return = 100 * (close - open) / open") + +// Filter Q4 2022 — this is the baseline comparison window +q4_2022 = FilterStream(returns, "date >= '2022-10-01' and date <= '2022-12-31'") + +// Filter Q4 2023 — the evaluation period +q4_2023 = FilterStream(returns, "date >= '2023-10-01' and date <= '2023-12-31'") + +// Compute the average daily return in Q4 2022 +// GroupByStream preserves all existing columns and adds `daily_return_mean` +avg_q4_2022 = GroupByStream(q4_2022, "", "daily_return:Mean") + +// Join each Q4 2023 row with the global Q4 2022 average (cartesian join) +// Because both inputs contain `daily_return`, prefixing is applied +joined = JoinStream(q4_2023, "", avg_q4_2022, "") + +// Clean up schema: +// - `left_daily_return` is the Q4 2023 value +// - `daily_return_mean` is preserved from the right side (Q4 2022 average) +// No `right_daily_return_mean` is needed — no collision occurred +renamed = ProjectStream(joined, " + left_date:date, + left_daily_return:q4_return, + daily_return_mean:q4_2022_avg_return +") + +// Compute Q4 2023 return as a percentage of the Q4 2022 average +relative = SingleRowMathStream(renamed, "relative_return = q4_return / q4_2022_avg_return") + +// Final output with clean, contextualized names +result = ProjectStream(relative, "date, q4_return, q4_2022_avg_return, relative_return") +``` + +--- + +### Example G4 — What Happened to ZN Price and Volume After the Last 10 FOMC Rate Decisions? + +**NL Query:** +What did the volume and price of ZN do in the 5 minutes following the last 10 FOMC rates? + +``` +// Load 5-minute windows immediately after each FOMC announcement +bars_1 = FutureHistoricalBarsStream("ZN", "ONE_MIN", "2023-11-01T14:00:00", "2023-11-01T14:05:00") +bars_2 = FutureHistoricalBarsStream("ZN", "ONE_MIN", "2023-09-20T14:00:00", "2023-09-20T14:05:00") +bars_3 = FutureHistoricalBarsStream("ZN", "ONE_MIN", "2023-07-26T14:00:00", "2023-07-26T14:05:00") +bars_4 = FutureHistoricalBarsStream("ZN", "ONE_MIN", "2023-06-14T14:00:00", "2023-06-14T14:05:00") +bars_5 = FutureHistoricalBarsStream("ZN", "ONE_MIN", "2023-05-03T14:00:00", "2023-05-03T14:05:00") +bars_6 = FutureHistoricalBarsStream("ZN", "ONE_MIN", "2023-03-22T14:00:00", "2023-03-22T14:05:00") +bars_7 = FutureHistoricalBarsStream("ZN", "ONE_MIN", "2023-02-01T14:00:00", "2023-02-01T14:05:00") +bars_8 = FutureHistoricalBarsStream("ZN", "ONE_MIN", "2022-12-14T14:00:00", "2022-12-14T14:05:00") +bars_9 = FutureHistoricalBarsStream("ZN", "ONE_MIN", "2022-11-02T14:00:00", "2022-11-02T14:05:00") +bars_10 = FutureHistoricalBarsStream("ZN", "ONE_MIN", "2022-09-21T14:00:00", "2022-09-21T14:05:00") + +// Combine all event windows into a single stream +all_bars = UnionStream(bars_1, bars_2) +all_bars = UnionStream(all_bars, bars_3) +all_bars = UnionStream(all_bars, bars_4) +all_bars = UnionStream(all_bars, bars_5) +all_bars = UnionStream(all_bars, bars_6) +all_bars = UnionStream(all_bars, bars_7) +all_bars = UnionStream(all_bars, bars_8) +all_bars = UnionStream(all_bars, bars_9) +all_bars = UnionStream(all_bars, bars_10) + +// Group by date (each FOMC window) and compute summary stats +grouped_data = GroupByStream(all_bars, "date", "open:First, close:Last, high:Max, low:Min, volume:Sum", "ts_event") + +// Rename columns for output readability +result = ProjectStream(grouped_data, "ts_event_date, open_first:5_min_open, close_last:5_min_close, high_max:5_min_high, low_min:5_min_low, volume_sum:5_min_volume") +``` + +--- + +### Example G5 — Daily RBOB Volume During the Settlement Window + +**NL Query:** +How much volume of RBOB typically trades during the settlement window? + +``` +// Load 1-minute bars for RBOB futures across the full year of 2023 +rbob_bars = FutureHistoricalBarsStream("RB", "ONE_MIN", "2023-01-01T00:00:00", "2023-12-31T23:59:59") + +// Expand datetime into hour and minute components for time filtering +expanded_bars = DateTimeExpanderStream(rbob_bars, "ts_event") + +// Select bars in the RBOB settlement window: 14:28 to <14:30 CST +settlement_window = FilterStream(expanded_bars, "(ts_event_hour = 14 and ts_event_minute >= 28) or (ts_event_hour = 14 and ts_event_minute < 30)") + +// Group by day and sum volume to compute daily settlement activity +typical_settlement_volume = GroupByStream(settlement_window, "ts_event_date", "volume:Sum") + +// Rename column for output clarity +result = ProjectStream(typical_settlement_volume, "ts_event_date, volume_sum:daily_settlement_volume") +``` + +--- + +### Example G6 — Average SP Futures Volume During Settlement Window in Pre-Expiry Weeks (2023) + +**NL Query:** +How much SP Futures trade during the settlement window, average per day, during the 2-week windows before future expiration, in 2023? + +``` +// Load 1-minute bars for S&P 500 futures for the full year of 2023 +sp_bars = FutureHistoricalBarsStream("ES", "ONE_MIN", "2023-01-01T00:00:00", "2023-12-31T23:59:59") + +// Add calendar breakdown (date, hour, minute, etc.) +expanded_bars = DateTimeExpanderStream(sp_bars, "ts_event") + +// Filter to settlement window: 15:58–16:00 CST +settlement_window = FilterStream(expanded_bars, "(ts_event_hour = 15 and ts_event_minute >= 58) or (ts_event_hour = 16 and ts_event_minute < 1)") + +// Further filter to the 2-week periods leading up to quarterly futures expiration dates +filtered_bars = FilterStream(settlement_window, " + (ts_event_date >= '2023-03-03' and ts_event_date < '2023-03-17') or + (ts_event_date >= '2023-06-02' and ts_event_date < '2023-06-16') or + (ts_event_date >= '2023-09-01' and ts_event_date < '2023-09-15') or + (ts_event_date >= '2023-12-01' and ts_event_date < '2023-12-15') +") + +// Aggregate daily settlement volume within those windows +daily_volume = GroupByStream(filtered_bars, "ts_event_date", "volume:Sum") + +// Compute the average of those daily volumes +average_daily_volume = GroupByStream(daily_volume, "", "volume_sum:Mean") + +// Rename for final output clarity +result = ProjectStream(average_daily_volume, "volume_sum_mean:average_daily_volume") +``` + +--- + +### Example G7 — Count of 2% Up vs. Down Days in SP500 Futures (2023) + +**NL Query:** +Find the difference between the number of days in 2023 that SP 500 went up 2% and when it went down 2%. + +``` +// Load daily bars for SP500 futures for calendar year 2023 +bars = FutureHistoricalBarsStream("ES", "ONE_DAY", "2023-01-01T00:00:00", "2023-12-31T23:59:59") + +// Compute daily percent change from open to close +percentage_change = SingleRowMathStream(bars, "percent_change = 100 * (close - open) / open") + +// Filter for days with +2% or more gain +days_up_2_percent = FilterStream(percentage_change, "percent_change >= 2") + +// Count those up days +days_up_count = CountStream(days_up_2_percent) + +// Filter for days with -2% or more loss +days_down_2_percent = FilterStream(percentage_change, "percent_change <= -2") + +// Count those down days +days_down_count = CountStream(days_down_2_percent) + +// Join both counts using a cartesian join (no join keys) +joined_counts = JoinStream(days_up_count, "", days_down_count, "") + +// Rename prefix-labeled columns for clarity +joined_counts_cleaned = ProjectStream(joined_counts, "left_count:days_up_count, right_count:days_down_count") + +// Compute the difference between up and down day counts +difference_result = SingleRowMathStream(joined_counts_cleaned, "difference = days_up_count - days_down_count") +``` + +--- + +### Example G8 — Monitor CL Contracts with Large Bid Sizes (Live Streaming) + +**NL Query:** +Monitor which CL contracts are trading with bid size ≥ 1000, and record the last 1000 events. + +``` +// Stream live best-bid-offer quotes across all CL contracts (wildcard match) +bbo = LiveBBOStream("CL*") + +// Filter to retain only rows where bid size is 1000 or more +large_bid = FilterStream(bbo, "bid_size >= 1000") + +// Persist last 1000 qualifying events as a time series buffer +large_bid_ts = TimeSeriesStream(large_bid, 1000) + +// Project selected columns for downstream use or display +result = ProjectStream(large_bid_ts, "timestamp, ts_event, contract, bid_size, bid_price, ask_price") +``` + +--- + +### Example G9 — Identify FutureSpreads in Crude Oil and Monitor Their Width + +**NL Query:** +Identify FutureSpreads in CrudeOil and monitor their spread width. + +``` +// Stream live bid/ask quotes across all CL contracts (including spreads) +bbo = LiveBBOStream("CL*") + +// Enrich with contract metadata (e.g., contract_type) +expanded_bbo = ContractExpanderStream(bbo, "contract") + +// Filter to retain only rows classified as future spreads +future_spreads = FilterStream(expanded_bbo, "contract_type = 'FutureSpread'") + +// Compute real-time spread width: ask - bid +spread_stream = SingleRowMathStream(future_spreads, "spread = ask_price - bid_price") + +// Output timestamp, contract, and computed spread width +result = ProjectStream(spread_stream, "ts_event, contract, spread") +``` + +--- + +### Example G10 — Track Natural Gas Bid Price Cross Above 3.00 (Last 10 Seconds Only) + +**NL Query:** +Track when bid prices in NaturalGas Futures cross above 3.00 and keep only the last 10 seconds of such events. + +``` +// Stream live bid/ask quotes for Natural Gas futures +bbo = LiveBBOStream("NG") + +// Detect when bid price crosses above the 3.00 threshold +high_bid = FilterStream(bbo, "bid_price > 3.00") + +// Persist all qualifying events into a time series buffer +high_bid_ts = TimeSeriesStream(high_bid, 10000) + +// Keep only those events that occurred in the last 10 seconds +recent_high_bid = TimeBasedRollingFilterStream(high_bid_ts, "ts_event", 10) + +// Output relevant quote data for each crossing event +result = ProjectStream(recent_high_bid, "ts_event, contract, bid_price, bid_size, ask_price") +``` + +--- + +### Example G11 — Detect Sequences of 3 Consecutive Bid Increases in Natural Gas Futures + +**NL Query:** +Detect sequences in NaturalGas futures where `bid_price` increased 3 times consecutively within a rolling 30-second window. + +``` +// Stream live bid/ask quotes for Natural Gas futures +bbo = LiveBBOStream("NG") + +// Persist all live BBO events to enable historical context +bbo_ts = TimeSeriesStream(bbo, 10000) + +// Limit to the most recent 30 seconds of events (rolling time window) +recent_bbo = TimeBasedRollingFilterStream(bbo_ts, "ts_event", 30) + +// Use Python to detect 3 consecutive upward ticks in bid_price +result = PythonDataframeStream(recent_bbo, " +def transform(df): + import pandas as pd + df = df.sort_values('ts_event') + + # Identify upward changes in bid_price + df['bid_increase'] = df['bid_price'].diff() > 0 + + # Rolling count of increases over 3 rows + df['consec_increase'] = df['bid_increase'].rolling(window=3).sum() + + # Trigger only when we just hit 3 consecutive increases + df['trigger'] = (df['consec_increase'] == 3) & (df['consec_increase'].shift(1) < 3) + + return df[df['trigger']][['ts_event', 'contract', 'bid_price', 'bid_size']] +") +``` + +--- + +### Example G12 — Track Post-Sweep Bid Behavior in SP500 Futures + +**NL Query:** +When we get a 5+ level bid sweep in SP 500, show me how the bid behaves in the 10 minutes after — mean, stddev, min/max bid\_price. + +``` +// Stream live bid/ask quotes for SP500 futures +bbo = LiveBBOStream("ES") + +// Maintain a rolling buffer of all BBO events (for real-time processing) +bbo_ts = TimeSeriesStream(bbo, 10000) + +// Maintain an independent stream with event windowing for sweep tracking +recent_bbo = TimeSeriesStream(bbo, 5000) + +result = PythonDataframeStream(recent_bbo, " +def transform(df): + import pandas as pd + df = df.sort_values('ts_event') + rows = [] + in_measurement = False + measurement_start_time = None + measurement_end_time = None + bid_prices_in_window = [] + min_bid_size_threshold = 100 # minimum threshold to consider a bid as 'stable' + min_levels_stable = 5 # number of stable levels required before collapse + prior_bid_sizes = [] + + for i, row in df.iterrows(): + bid_price = row['bid_price'] + bid_size = row['bid_size'] + ts = pd.to_datetime(row['ts_event']) + + # Maintain rolling size buffer + prior_bid_sizes.append(bid_size) + if len(prior_bid_sizes) > min_levels_stable: + prior_bid_sizes.pop(0) + + if not in_measurement: + # Trigger on a drop after consecutive large bid sizes + if all(x > min_bid_size_threshold for x in prior_bid_sizes[:-1]) and bid_size < min_bid_size_threshold: + measurement_start_time = ts + measurement_end_time = measurement_start_time + pd.Timedelta(minutes=10) + bid_prices_in_window = [] + in_measurement = True + else: + # Still inside the 10-minute measurement window + if ts <= measurement_end_time: + bid_prices_in_window.append(bid_price) + else: + # End of measurement window → emit statistics + if bid_prices_in_window: + mean_bid = pd.Series(bid_prices_in_window).mean() + stddev_bid = pd.Series(bid_prices_in_window).std() + min_bid = pd.Series(bid_prices_in_window).min() + max_bid = pd.Series(bid_prices_in_window).max() + rows.append({ + 'sweep_time': measurement_start_time, + 'measurement_end': measurement_end_time, + 'mean_bid_price': mean_bid, + 'stddev_bid_price': stddev_bid, + 'min_bid_price': min_bid, + 'max_bid_price': max_bid + }) + # Reset state + in_measurement = False + measurement_start_time = None + measurement_end_time = None + bid_prices_in_window = [] + prior_bid_sizes = [] + + # Always return a DataFrame with correct schema + columns = [ + 'sweep_time', + 'measurement_end', + 'mean_bid_price', + 'stddev_bid_price', + 'min_bid_price', + 'max_bid_price' + ] + return pd.DataFrame(rows, columns=columns) +") +``` + +--- + +### Example G13 — Compute Daily VWAP for ZN Over the Last 60 Sessions + +### **NL Query:** What was the daily VWAP for ZN over the last 60 sessions? + +``` +// Load 1-minute bars for ZN covering the last 60 trading days +bars = FutureHistoricalBarsStream("ZN", "ONE_MIN", "2024-03-15T00:00:00", "2024-06-13T23:59:59") + +// Compute per-bar dollar value (price × volume) +dollar_value = SingleRowMathStream(bars, "dollar_value = close * volume") + +// Extract calendar date to group by session +expanded = DateTimeExpanderStream(dollar_value, "ts_event") + +// Group by trading day and compute total dollar value and total volume +daily_sums = GroupByStream(expanded, "ts_event_date", "dollar_value:Sum, volume:Sum") + +// Compute daily VWAP: total dollar value divided by total volume +vwap = SingleRowMathStream(daily_sums, "vwap = dollar_value_sum / volume_sum") + +// Return daily VWAP alongside date +result = ProjectStream(vwap, "ts_event_date, vwap") +``` + +### Example G14 — Top 3 Highest Volume Days in ZN (2024) + +### **NL Query:** What are the 3 highest volume days for ZN in 2024? + +### + +``` +// Load daily bars for ZN futures in calendar year 2024 +bars = FutureHistoricalBarsStream("ZN", "ONE_DAY", "2024-01-01T00:00:00", "2024-12-31T23:59:59") + +// Retain only the date and volume columns for ranking +bars_cleaned = ProjectStream(bars, "date, volume") + +// Select the top 3 rows sorted by descending volume +top_3 = TopStream(bars_cleaned, 3, "-volume") +``` + +### + +--- + +### Example G15 — Stream Synthetic Spread Book: 2.3×ES vs. 1×NQ + +### **NL Query:** Stream the level 1 book of a synthetic spread defined as 2.3 × E-mini S\&P 500 futures vs 1 E-mini NASDAQ-100 future. + +### + +``` +// Stream best bid/ask quotes for E-mini S&P 500 futures +bbo_es = LiveBBOStream("ES") +// Stream best bid/ask quotes for E-mini NASDAQ-100 futures +bbo_nq = LiveBBOStream("NQ") +// Perform a cartesian join (no keys) to pair latest ticks from both instruments +joined = JoinStream(bbo_es, "", bbo_nq, "") +// Compute synthetic spread bid/ask prices and tradable sizes +spread_book = SingleRowMathStream(joined, " + spread_bid = 2.3 * left_bid_price - 1 * right_ask_price; + spread_ask = 2.3 * left_ask_price - 1 * right_bid_price; + spread_bid_size = Min(left_bid_size, right_ask_size); + spread_ask_size = Min(left_ask_size, right_bid_size) +") +// Output the constructed spread book +result = ProjectStream(spread_book, "ts_event, spread_bid, spread_bid_size, spread_ask, spread_ask_size") +``` + +### + +### --- + +### + +### Example 16 — Compute the rolling one-minute 99% Value-at-Risk for a 100-lot position in 10-year Treasury Note futures using an exponentially weighted volatility estimate over the past two hours, and trigger an alert if VaR exceeds $100,000. + +### + +``` +trades = LiveTradeStream("ZN") +recent = TimeBasedRollingFilterStream(trades, "ts_event", 7200) +ew_var = PythonDataframeStream(recent, " +def transform(df): + import pandas as pd + df = df.sort_values('ts_event') + df['log_return'] = (df['trade_price'] / df['trade_price'].shift(1)).apply(lambda x: pd.np.log(x)) + df['ew_vol'] = df['log_return'].ewm(span=7200).std() + df['var_99'] = 2.33 * df['ew_vol'] * 100 # 100-lot position + df = df[df['var_99'] > 100000] + return df[['ts_event', 'contract', 'var_99']] +") +result = ProjectStream(ew_var, "ts_event, contract, var_99") + + +``` + +## Appendix Z — Formal Grammar (EBNF) + +``` +(* ---------------------------------------------------------------- *) +(* Terminal types *) +(* ---------------------------------------------------------------- *) +StreamIdentifier ::= IDENTIFIER (* previously-defined stream name *) +ColumnName ::= IDENTIFIER +ISODateTime ::= STRING_LITERAL (* "2023-01-01T00:00:00" etc. *) +JsonText ::= STRING_LITERAL (* JSON array of records *) +ContractSymbol ::= STRING_LITERAL (* "CL", "ZN*", "ESZ9", … *) +BarSize ::= "ONE_DAY" | "ONE_MIN" + +(* ---------------------------------------------------------------- *) +(* Data-source streams *) +(* ---------------------------------------------------------------- *) +FutureHistoricalBarsStream ::= + "FutureHistoricalBarsStream" "(" ContractSymbol "," BarSize + [ "," ISODateTime [ "," ISODateTime ] ] ")" + +LiveBBOStream ::= + "LiveBBOStream" "(" ContractSymbol ")" + +LiveTradeStream ::= + "LiveTradeStream" "(" ContractSymbol ")" + +(* ---------------------------------------------------------------- *) +(* Operator streams *) +(* ---------------------------------------------------------------- *) +FilterStream ::= + "FilterStream" "(" StreamIdentifier "," ConditionExpr ")" +ConditionExpr ::= STRING_LITERAL (* e.g. "volume > 1000 and contract = 'CL'" *) + +ProjectStream ::= + "ProjectStream" "(" StreamIdentifier "," ColumnMap ")" +ColumnMap ::= STRING_LITERAL (* e.g. "close:settlement, volume" *) + +GroupByStream ::= + "GroupByStream" "(" StreamIdentifier "," GroupKey "," AggregationList ")" +GroupKey ::= STRING_LITERAL (* "" for global or comma-separated keys *) +AggregationList ::= Aggregation { "," Aggregation } +Aggregation ::= ColumnName ":" AggregationFunction +AggregationFunction ::= + "Sum" | "Mean" | "Median" | "Mode" | "Max" | "Min" + | "StdDev" | "Range" | "First" | "Last" | "EdgeDiff" | "EdgeDiffPercentage" + +SingleRowMathStream ::= + "SingleRowMathStream" "(" StreamIdentifier "," MathExpression ")" +MathExpression ::= STRING_LITERAL (* e.g. "vwap = close * volume" *) + +MultiRowRollingMathStream ::= + "MultiRowRollingMathStream" "(" StreamIdentifier "," AggregationList "," SortColumn "," WindowSize ")" +SortColumn ::= "+" ColumnName | "-" ColumnName +WindowSize ::= INTEGER (* number of rows *) + +UnionStream ::= + "UnionStream" "(" StreamIdentifier "," StreamIdentifier ")" + +JoinStream ::= + "JoinStream" "(" LeftStream "," LeftJoinKeys "," RightStream "," RightJoinKeys ")" +LeftStream ::= StreamIdentifier +RightStream ::= StreamIdentifier +LeftJoinKeys ::= STRING_LITERAL (* comma-separated or "" *) +RightJoinKeys ::= STRING_LITERAL + +DateTimeExpanderStream ::= + "DateTimeExpanderStream" "(" StreamIdentifier "," DatetimeColumn ")" +DatetimeColumn ::= STRING_LITERAL (* e.g. "ts_event" *) + +ContractExpanderStream ::= + "ContractExpanderStream" "(" StreamIdentifier "," ContractColumn ")" +ContractColumn ::= STRING_LITERAL + +CountStream ::= + "CountStream" "(" StreamIdentifier ")" + +TimeSeriesStream ::= + "TimeSeriesStream" "(" StreamIdentifier "," BufferSize ")" +BufferSize ::= INTEGER + +TimeBasedRollingFilterStream ::= + "TimeBasedRollingFilterStream" "(" StreamIdentifier "," TimeColumn "," Seconds ")" +TimeColumn ::= STRING_LITERAL +Seconds ::= INTEGER + +PythonDataframeStream ::= + "PythonDataframeStream" "(" StreamIdentifier "," PythonFunction ")" +PythonFunction ::= STRING_LITERAL (* full Python function as a string *) + +TopStream ::= + "TopStream" "(" StreamIdentifier "," NumRows "," SortColumns ")" +NumRows ::= INTEGER +SortColumns ::= STRING_LITERAL (* e.g. "-volume, +contract" *) +``` + +#### + +#### --- + +## 8 Best Practices + +This section documents idiomatic BlueOrbDSL usage patterns and guidance. +While not syntactically required, these practices help ensure correctness, clarity, and reusability. + +### 8.1 Join Hygiene + +- **Avoid using `ts_event` as a join key**, especially with real-time streams. Use `ts_event_date` or `contract` instead. +- **All shared column names will be prefixed** (`left_`, `right_`), including join keys. Plan projections accordingly. +- **Always follow JoinStream with ProjectStream** to clarify or rename output columns. Only retain columns that will be used further downstream. + +### 8.2 Stream Composition + +- Order streams logically: **DataSource → Math → Filter → Group → Project**. +- Compute any derived expressions **before** using them in `FilterStream` or `GroupByStream`. +- Avoid redundant joins or filters — combine conditions early where possible. +- Do not use `SingleRowMathStream` for boolean conditions (e.g., `trigger = x > 1.0`). Use `FilterStream` instead. + +### 8.3 Naming & Readability + +- Use **descriptive stream variable names**: e.g., `price_movements`, `high_volume_trades`. +- Avoid overwriting streams unless intentional; reuse clearly named intermediates. +- Prefer `snake_case` for all column names and stream variables. + +### 8.4 PythonDataframeStream Usage + +- Use **only when no native DSL construct can express the logic**. +- Keep Python functions **small and well-documented** (\< 20 lines). +- Minimize use of external libraries; prefer pandas-native ops. + +### 8.5 TimeSeries Design + +- Use `TimeSeriesStream` early to **capture rolling context**, not retroactively. +- For rolling-time-window logic, combine: + `LiveStream → TimeSeriesStream → TimeBasedRollingFilterStream → GroupByStream`. + +### 8.6 Cleaning up end result + +- ✳️ Always consider adding a `ProjectStream` at the end of the pipeline to return **only the essential columns**, clearly renamed. This is especially important when: + - The final result involves aggregation (e.g., `log_return:Mean`) + - Intermediate columns (e.g., `ts_event`, `volume`, `open`) are no longer needed + - You want the output to be clean, minimal, and user-facing + +✅ Example: + +``` +result = ProjectStream(avg_return, "log_return_mean:avg_log_return") +``` + diff --git a/src/DataStreamSuite/SharedStreams/NL/BlueOrbNLStream.cs b/src/DataStreamSuite/SharedStreams/NL/BlueOrbNLStream.cs index 47f983024..48709114b 100644 --- a/src/DataStreamSuite/SharedStreams/NL/BlueOrbNLStream.cs +++ b/src/DataStreamSuite/SharedStreams/NL/BlueOrbNLStream.cs @@ -14,17 +14,19 @@ [DataStreamUserRespresentation(false)] public class BlueOrbNLStream : ViewStream { private readonly string _nl_prompt; + readonly string masterLearningPrompt; readonly IDataStreamFactory streamFactory; ViewStream? _nl_to_dsl_stream; ViewStream? _dsl_stream; private IDictionary _dp_metadata = new Dictionary(); - public BlueOrbNLStream(string nl_prompt, IDataStreamFactory streamFactory) { + public BlueOrbNLStream(string nl_prompt, string masterLearningPrompt, IDataStreamFactory streamFactory) { _nl_prompt = nl_prompt; + this.masterLearningPrompt = masterLearningPrompt; this.streamFactory = streamFactory; } protected override void myOnFirstSubscriber() { - _nl_to_dsl_stream = new BlueOrbNLToDSLStream(_nl_prompt); + _nl_to_dsl_stream = new BlueOrbNLToDSLStream(_nl_prompt, masterLearningPrompt); _nl_to_dsl_stream.onChange += _handle_dsl_callback; } diff --git a/src/DataStreamSuite/SharedStreams/NL/BlueOrbNLToDSLStream.cs b/src/DataStreamSuite/SharedStreams/NL/BlueOrbNLToDSLStream.cs index 46cfc81a5..0fe0eaa69 100644 --- a/src/DataStreamSuite/SharedStreams/NL/BlueOrbNLToDSLStream.cs +++ b/src/DataStreamSuite/SharedStreams/NL/BlueOrbNLToDSLStream.cs @@ -9,26 +9,9 @@ using System.Threading.Tasks; using System.Runtime.InteropServices; - [DataStreamUserRespresentation("BlueOrbNLToDSLStream")] + [DataStreamUserRespresentation(false)] public class BlueOrbNLToDSLStream : ViewStream { - public BlueOrbNLToDSLStream(string nl_prompt) { - string mlp_file_path = "C:\\Users\\arham\\Downloads\\" + "BlueOrbDSL_MLP_v1.9.md"; - if (RuntimeInformation.IsOSPlatform(OSPlatform.Windows)) { - // Use current user's Downloads folder - string downloadsPath = Path.Combine( - Environment.GetFolderPath(Environment.SpecialFolder.UserProfile), - "Downloads" - ); - mlp_file_path = Path.Combine(downloadsPath, "BlueOrbDSL_MLP_v1.9.md"); - } else { - // On Linux, use ~/bd/ - string home = Environment.GetEnvironmentVariable("HOME") ?? "/tmp"; - string bldPath = Path.Combine(home, "BlueOrb_Data"); - Directory.CreateDirectory(bldPath); // make sure it exists - mlp_file_path = Path.Combine(bldPath, "BlueOrbDSL_MLP_v1.9.md"); - } - - + public BlueOrbNLToDSLStream(string nl_prompt, string mlp_file_path) { string mlp_text = File.ReadAllText(mlp_file_path); string system_prompt = $"You are a deterministic BlueOrbDSL generator. " + $"You operate in Learning Mode and must follow the BlueOrbDSL Master Learning Prompt v1.9 strictly. " + diff --git a/src/DataStreamSuite/SharedStreams/NL/NLPromptConfig.cs b/src/DataStreamSuite/SharedStreams/NL/NLPromptConfig.cs deleted file mode 100644 index 27faaa1bb..000000000 --- a/src/DataStreamSuite/SharedStreams/NL/NLPromptConfig.cs +++ /dev/null @@ -1,77 +0,0 @@ -namespace bts.SilverPine; - -using Spark.Core.Json; - -using System.Collections.Generic; -using System.Text.Json; -using System.Text.Json.Serialization; - -public class NLPromptConfig { - static NLPromptConfig? instance; - - public static NLPromptConfig Instance => - instance ?? throw new InvalidOperationException( - "SetGlobalConfig must be called to initialize config static singleton"); - - public List prompts { get; } = new(0); - - public static void SetGlobalConfig(NLPromptConfig cfg) { - // This should only be called once - Log.isnull(instance); - instance = Log.notnull(cfg); - } - - public static NLPromptConfig Default() => new(); - - // Represents a pair of a natural language query and its dsl - public class FewShotExample { - public string? natural_language { get; set; } - public string? dsl { get; set; } - } - - public class FewShotPrompt { - public string? category { get; set; } - - public string? description { get; set; } - - public bool IsEnabled { get; set; } = true; - - // MultiLineDescription is simply here for convenience. It is functionally redundant, as it - // serves the same purpose as 'description'. If 'description' is too long, you can break it - // up into an array of strings and assign it to MultiLineDescription instead. - public List? MultiLineDescription { get; set; } - - // the actual prompt given by the user/CME - public string? prompt { get; set; } - - public List examples { get; } = new(0); - - public override string ToString() { - var desc = description ?? string.Concat(MultiLineDescription.empty_if_null()); - string rv = $"System: {desc}\n"; - int ex_no = 0; - foreach (FewShotExample ex in examples ?? new(0)) { - rv += $"{++ex_no}. Natural Language: \"{ex.natural_language}\"" + "\n"; - rv += $" DSL: {ex.dsl}\n"; - - } - return rv; - } - - // Reads normal strings and arrays of lines - public class FewShotPromptLinesConverter : JsonConverter { - public override string? Read(ref Utf8JsonReader reader, Type typeToConvert, JsonSerializerOptions options) { - if (reader.ReadString(out var str)) { - return str; - } - if (reader.ReadArray(out var array)) { - return string.Join('\n', array.Select(v => v?.AsString())); - } - throw new JsonException("Failed to parse FewShotPrompt lines"); - } - - public override void Write(Utf8JsonWriter writer, string value, JsonSerializerOptions options) => - writer.WriteStringValue(value); - } - } -} diff --git a/src/DataStreamSuite/SharedStreams/NL/NLStreamFactory.cs b/src/DataStreamSuite/SharedStreams/NL/NLStreamFactory.cs deleted file mode 100644 index 5e21ada4a..000000000 --- a/src/DataStreamSuite/SharedStreams/NL/NLStreamFactory.cs +++ /dev/null @@ -1,44 +0,0 @@ -namespace bts.SilverPine; - -using BCG; - -using bts.BlueOrb; - -using Bts.DataStreamCore; - -using mbts.utils; - -using System.Collections.Generic; - -public class NLStreamFactory : IRecursiveDataStreamFactory { - IDataStreamFactory? compositeFactory; - - public ViewStream? Create(string streamId, IEnumerable<(string? Name, int? Index, object? Value)> args) { - if (SupportedStreams.NaturalLanguage.IsMatch(streamId)) { - if (compositeFactory is null) { - Log.DataFlow.warn("Failed to create NLStream because 'compositeFactory' is null."); - return null; - } - if (args.SingleOrDefault().Value is string prompt) { - return new BlueOrbNLStream(prompt, compositeFactory); - } - } - return null; - } - - public IEnumerable GetSupportedStreams() { - yield return SupportedStreams.NaturalLanguage; - } - - public void OnCompositeFactory(IDataStreamFactory compositeFactory) => - this.compositeFactory = compositeFactory; - - static class SupportedStreams { - public static readonly DataStreamMetadata NaturalLanguage = new() { - Name = "nl", - AlternateNames = new() { "naturalLanguage" } - }; - - public static string NlName => ExceptionAssert.not_null(NaturalLanguage.Name); - } -} diff --git a/src/DataStreamSuite/SharedStreams/Services/SharedStreamsServices.cs b/src/DataStreamSuite/SharedStreams/Services/SharedStreamsServices.cs index 9b8c789b6..f6a118f45 100644 --- a/src/DataStreamSuite/SharedStreams/Services/SharedStreamsServices.cs +++ b/src/DataStreamSuite/SharedStreams/Services/SharedStreamsServices.cs @@ -16,14 +16,10 @@ public static class SharedStreamsServices { .addStreamFactory() .addStreamFactory(); - public static IServiceCollection AddNaturalLanguageStreams(this IServiceCollection services, IConfigurationSection configurationSection) { - services.Configure(configurationSection); - NLPromptConfig.SetGlobalConfig(configurationSection.Get() ?? NLPromptConfig.Default()); - // TODO: integrate config into a NLStream provider (so it can provide config values - // instead of the ViewStream's getting them from a static singleton) - return services.AddSingleton() - .AddSingleton(sp => sp.GetRequiredService()); - } + public static IServiceCollection AddNaturalLanguageStreams(this IServiceCollection services, IConfigurationSection configurationSection) => + services.Configure(configurationSection) + .AddSingleton() + .AddSingleton(sp => sp.GetRequiredService()); static IServiceCollection addStreamFactory(this IServiceCollection services) where T : class, IDataStreamFactory => services.AddSingleton() diff --git a/src/DataStreamSuite/SharedStreams/SharedStreams.csproj b/src/DataStreamSuite/SharedStreams/SharedStreams.csproj index 44f691af0..7c7a30d37 100644 --- a/src/DataStreamSuite/SharedStreams/SharedStreams.csproj +++ b/src/DataStreamSuite/SharedStreams/SharedStreams.csproj @@ -21,4 +21,8 @@ + + + + diff --git a/src/DataStreamSuite/SharedStreams/StreamFactories/NaturalLanguageStreamFactory.cs b/src/DataStreamSuite/SharedStreams/StreamFactories/NaturalLanguageStreamFactory.cs new file mode 100644 index 000000000..e887ba8eb --- /dev/null +++ b/src/DataStreamSuite/SharedStreams/StreamFactories/NaturalLanguageStreamFactory.cs @@ -0,0 +1,39 @@ +namespace Bts.SharedStreams.StreamFactories; + +using BCG; + +using bts.BlueOrb; + +using Bts.DataStreamCore; + +using Microsoft.Extensions.Logging; +using Microsoft.Extensions.Options; + +public class NaturalLanguageStreamFactory : DataStreamFactory, IRecursiveDataStreamFactory { + IDataStreamFactory? compositeStreamFactory; + + public NaturalLanguageStreamFactory(IOptions options, ILogger logger) + : this(options.Value, logger) { } + + public NaturalLanguageStreamFactory(NaturalLanguageStreamFactoryConfig config, ILogger logger) { + this.Add("nl", CreateNaturalLanguage, logger); + + ViewStream? CreateNaturalLanguage(NaturalLanguageStreamParams streamParams) { + if (compositeStreamFactory is null) { + return null; + } + return new BlueOrbNLStream( + streamParams.Prompt ?? string.Empty, + config.MasterLearningPrompt ?? string.Empty, + compositeStreamFactory); + } + } + + public void OnCompositeFactory(IDataStreamFactory compositeStreamFactory) => + this.compositeStreamFactory = compositeStreamFactory; + + class NaturalLanguageStreamParams { + [StreamArgument(Position = 0)] + public string? Prompt { get; set; } + } +} diff --git a/src/DataStreamSuite/SharedStreams/StreamFactories/NaturalLanguageStreamFactoryConfig.cs b/src/DataStreamSuite/SharedStreams/StreamFactories/NaturalLanguageStreamFactoryConfig.cs new file mode 100644 index 000000000..9459ff3b2 --- /dev/null +++ b/src/DataStreamSuite/SharedStreams/StreamFactories/NaturalLanguageStreamFactoryConfig.cs @@ -0,0 +1,5 @@ +namespace Bts.SharedStreams.StreamFactories; + +public class NaturalLanguageStreamFactoryConfig { + public string? MasterLearningPrompt { get; set; } +}