From 94e9c8c3f287c155f12a1faabd6a0c73623398ef Mon Sep 17 00:00:00 2001 From: John Colvin Date: Fri, 31 Oct 2025 12:37:00 -0400 Subject: [PATCH] improve: Generate MLP at runtime with list of example files Adds prompt file list to NaturalLanguageStreamFactoryConfig --- .../SharedStreams/SharedStreams.csproj | 3 + .../NaturalLanguageStreamFactory.cs | 95 +++++++++++++++++++ .../NaturalLanguageStreamFactoryConfig.cs | 1 + src/Directory.Build.props | 14 +++ src/Directory.Build.targets | 34 +++++++ 5 files changed, 147 insertions(+) diff --git a/src/DataStreamSuite/SharedStreams/SharedStreams.csproj b/src/DataStreamSuite/SharedStreams/SharedStreams.csproj index f4dcc4618..f5864b49f 100644 --- a/src/DataStreamSuite/SharedStreams/SharedStreams.csproj +++ b/src/DataStreamSuite/SharedStreams/SharedStreams.csproj @@ -20,10 +20,13 @@ + + + diff --git a/src/DataStreamSuite/SharedStreams/StreamFactories/NaturalLanguageStreamFactory.cs b/src/DataStreamSuite/SharedStreams/StreamFactories/NaturalLanguageStreamFactory.cs index 5d916914e..bf56d0deb 100644 --- a/src/DataStreamSuite/SharedStreams/StreamFactories/NaturalLanguageStreamFactory.cs +++ b/src/DataStreamSuite/SharedStreams/StreamFactories/NaturalLanguageStreamFactory.cs @@ -2,13 +2,19 @@ namespace Bts.SharedStreams.StreamFactories; using BCG; +using bts; using bts.BlueOrb; +using bts.collections; using Bts.DataStreamCore; using Microsoft.Extensions.Logging; using Microsoft.Extensions.Options; +using System.Text; + +using YamlDotNet.RepresentationModel; + public class NaturalLanguageStreamFactory : DataStreamFactory, IRecursiveDataStreamFactory { IDataStreamFactory? compositeStreamFactory; @@ -16,6 +22,39 @@ public class NaturalLanguageStreamFactory : DataStreamFactory, IRecursiveDataStr : this(options.Value, viewStreamDatabase, logger) { } public NaturalLanguageStreamFactory(NaturalLanguageStreamFactoryConfig config, ViewStreamDatabase viewStreamDatabase, ILogger logger) { + string prompt = config.MasterLearningPrompt ?? string.Empty; + if (config.MasterLearningPromptExamples?.Count > 0) { + var dir = Path.GetDirectoryName(prompt); + Log.notnull(dir); + var promptText = File.ReadAllLines(prompt); + prompt = Path.Combine(dir, Path.GetFileNameWithoutExtension(config.MasterLearningPrompt) + "_generated.md"); + var examplesBd = new StringBuilder(); + int exNum = 0; + foreach (var e in config.MasterLearningPromptExamples.Select(loadExample).ExceptNull()) { + exNum++; + examplesBd.AppendLine($"### Example G{exNum} - {e.Description}"); + examplesBd.AppendLine(); + examplesBd.AppendLine("**NL Query:**"); + examplesBd.AppendLine(e.NaturalLanguage); + examplesBd.AppendLine(); + examplesBd.AppendLine("```"); + examplesBd.AppendLine(e.SampleCode); + examplesBd.AppendLine("```"); + examplesBd.AppendLine(); + examplesBd.AppendLine("---"); + examplesBd.AppendLine(); + } + File.WriteAllLines(prompt, promptText); + using var fw = File.OpenWrite(prompt); + using var sw = new StreamWriter(fw); + foreach (var line in promptText) { + if (string.Equals(line, "{EXAMPLES}", StringComparison.CurrentCultureIgnoreCase)) { + sw.WriteLine(examplesBd.ToString()); + } else { + sw.WriteLine(line); + } + } + } this.Add("nl", CreateNaturalLanguage, logger); ViewStream? CreateNaturalLanguage(NaturalLanguageStreamParams streamParams) { @@ -35,8 +74,64 @@ public class NaturalLanguageStreamFactory : DataStreamFactory, IRecursiveDataStr public void OnCompositeFactory(IDataStreamFactory compositeStreamFactory) => this.compositeStreamFactory = compositeStreamFactory; + static PromptExample? loadExample(string file) { + string text = File.ReadAllText(file); + + // Extract YAML frontmatter + var parts = text.Split(new string[] { "---" }, StringSplitOptions.None); + if (parts.Length < 3) { + Console.WriteLine("No frontmatter found."); + return null; + } + + string yaml = parts[1]; // frontmatter is always between the first two '---' + string body = string.Join("---", parts.Skip(2)); // the rest is the markdown body + + // Parse YAML + var yamlStream = new YamlStream(); + yamlStream.Load(new StringReader(yaml)); + var root = (YamlMappingNode)yamlStream.Documents[0].RootNode; + var rv = new PromptExample(); + + // Get "desc" + if (root.Children.TryGetValue(new YamlScalarNode("desc"), out var descNode)) { + rv.Description = descNode.ToString(); + } + + // Get "desc" + if (root.Children.TryGetValue(new YamlScalarNode("nl"), out var nlNode)) { + rv.NaturalLanguage = nlNode.ToString(); + } + // Optionally, get the fenced code block + string dslCode = extractCodeBlock(body, "dsl"); + rv.SampleCode = dslCode; + return rv; + } + + static string extractCodeBlock(string markdown, string language) { + string fence = "```" + language; + int start = markdown.IndexOf(fence); + if (start == -1) { + return string.Empty; + } + + start += fence.Length; + int end = markdown.IndexOf("```", start); + if (end == -1) { + return string.Empty; + } + + return markdown[start..end].Trim(); + } + class NaturalLanguageStreamParams { [StreamArgument(Position = 0)] public string? Prompt { get; set; } } + + class PromptExample { + public string? Description { get; set; } + public string? NaturalLanguage { get; set; } + public string? SampleCode { get; set; } + } } diff --git a/src/DataStreamSuite/SharedStreams/StreamFactories/NaturalLanguageStreamFactoryConfig.cs b/src/DataStreamSuite/SharedStreams/StreamFactories/NaturalLanguageStreamFactoryConfig.cs index 7b6c66744..aec044484 100644 --- a/src/DataStreamSuite/SharedStreams/StreamFactories/NaturalLanguageStreamFactoryConfig.cs +++ b/src/DataStreamSuite/SharedStreams/StreamFactories/NaturalLanguageStreamFactoryConfig.cs @@ -4,4 +4,5 @@ public class NaturalLanguageStreamFactoryConfig { public string? ApiKey { get; set; } public string? MasterLearningPrompt { get; set; } public string? ModelName { get; set; } + public List? MasterLearningPromptExamples { get; set; } } diff --git a/src/Directory.Build.props b/src/Directory.Build.props index a7e2e217d..1eea2b915 100644 --- a/src/Directory.Build.props +++ b/src/Directory.Build.props @@ -95,6 +95,20 @@ $(OutputPath)\publish\Configurations\$(MSBuildProjectName) $(OutputPath)\publish\$(MSBuildProjectName).%(FileName)%(Extension) + + + $(OutputPath)\BlueOrbPrompts\$(MSBuildProjectName) + $(OutputPath)\$(MSBuildProjectName).%(FileName)%(Extension) + $(OutputPath)\publish\BlueOrbPrompts\$(MSBuildProjectName) + $(OutputPath)\publish\$(MSBuildProjectName).%(FileName)%(Extension) + + + + $(OutputPath)\BlueOrbPrompts\Examples\$(MSBuildProjectName) + $(OutputPath)\$(MSBuildProjectName).%(FileName)%(Extension) + $(OutputPath)\publish\BlueOrbPrompts\Examples\$(MSBuildProjectName) + $(OutputPath)\publish\$(MSBuildProjectName).%(FileName)%(Extension) + diff --git a/src/Directory.Build.targets b/src/Directory.Build.targets index 6ea2693db..5a0d31b2c 100644 --- a/src/Directory.Build.targets +++ b/src/Directory.Build.targets @@ -65,6 +65,26 @@ + + + + + + + + + +