improve: Generate MLP at runtime with list of example files

Adds prompt file list to NaturalLanguageStreamFactoryConfig
This commit is contained in:
John Colvin committed 2025-11-15 12:46:24 -05:00
1 parent 0f9e515d01
commit 94e9c8c3f2
5 files changed
+147

No files matched your search

@@ -20,10 +20,13 @@
<PackageReference Include="OpenAI" Version="2.0.0" />
<PackageReference Include="System.Linq.Async" Version="6.0.1" />
<PackageReference Include="Parquet.Net" Version="5.2.0" />
<PackageReference Include="YamlDotNet" Version="16.3.0" />
</ItemGroup>
<ItemGroup>
<BlueOrbPrompt Include="BlueOrb\Prompts\BlueOrbDSL_MLP_v1.9.md" />
<BlueOrbPrompt Include="BlueOrb\Prompts\BrandonChenMLP.md" />
<BlueOrbPromptExample Include="BlueOrb\Prompts\Examples\**\*.md" />
</ItemGroup>
</Project>
@@ -2,13 +2,19 @@ namespace Bts.SharedStreams.StreamFactories;
using BCG;
using bts;
using bts.BlueOrb;
using bts.collections;
using Bts.DataStreamCore;
using Microsoft.Extensions.Logging;
using Microsoft.Extensions.Options;
using System.Text;
using YamlDotNet.RepresentationModel;
public class NaturalLanguageStreamFactory : DataStreamFactory, IRecursiveDataStreamFactory {
IDataStreamFactory? compositeStreamFactory;
@@ -16,6 +22,39 @@ public class NaturalLanguageStreamFactory : DataStreamFactory, IRecursiveDataStr
: this(options.Value, viewStreamDatabase, logger) { }
public NaturalLanguageStreamFactory(NaturalLanguageStreamFactoryConfig config, ViewStreamDatabase viewStreamDatabase, ILogger<NaturalLanguageStreamFactory> logger) {
string prompt = config.MasterLearningPrompt ?? string.Empty;
if (config.MasterLearningPromptExamples?.Count > 0) {
var dir = Path.GetDirectoryName(prompt);
Log.notnull(dir);
var promptText = File.ReadAllLines(prompt);
prompt = Path.Combine(dir, Path.GetFileNameWithoutExtension(config.MasterLearningPrompt) + "_generated.md");
var examplesBd = new StringBuilder();
int exNum = 0;
foreach (var e in config.MasterLearningPromptExamples.Select(loadExample).ExceptNull()) {
exNum++;
examplesBd.AppendLine($"### Example G{exNum} - {e.Description}");
examplesBd.AppendLine();
examplesBd.AppendLine("**NL Query:**");
examplesBd.AppendLine(e.NaturalLanguage);
examplesBd.AppendLine();
examplesBd.AppendLine("```");
examplesBd.AppendLine(e.SampleCode);
examplesBd.AppendLine("```");
examplesBd.AppendLine();
examplesBd.AppendLine("---");
examplesBd.AppendLine();
}
File.WriteAllLines(prompt, promptText);
using var fw = File.OpenWrite(prompt);
using var sw = new StreamWriter(fw);
foreach (var line in promptText) {
if (string.Equals(line, "{EXAMPLES}", StringComparison.CurrentCultureIgnoreCase)) {
sw.WriteLine(examplesBd.ToString());
} else {
sw.WriteLine(line);
}
}
}
this.Add<NaturalLanguageStreamParams>("nl", CreateNaturalLanguage, logger);
ViewStream? CreateNaturalLanguage(NaturalLanguageStreamParams streamParams) {
@@ -35,8 +74,64 @@ public class NaturalLanguageStreamFactory : DataStreamFactory, IRecursiveDataStr
public void OnCompositeFactory(IDataStreamFactory compositeStreamFactory) =>
this.compositeStreamFactory = compositeStreamFactory;
static PromptExample? loadExample(string file) {
string text = File.ReadAllText(file);
// Extract YAML frontmatter
var parts = text.Split(new string[] { "---" }, StringSplitOptions.None);
if (parts.Length < 3) {
Console.WriteLine("No frontmatter found.");
return null;
}
string yaml = parts[1]; // frontmatter is always between the first two '---'
string body = string.Join("---", parts.Skip(2)); // the rest is the markdown body
// Parse YAML
var yamlStream = new YamlStream();
yamlStream.Load(new StringReader(yaml));
var root = (YamlMappingNode)yamlStream.Documents[0].RootNode;
var rv = new PromptExample();
// Get "desc"
if (root.Children.TryGetValue(new YamlScalarNode("desc"), out var descNode)) {
rv.Description = descNode.ToString();
}
// Get "desc"
if (root.Children.TryGetValue(new YamlScalarNode("nl"), out var nlNode)) {
rv.NaturalLanguage = nlNode.ToString();
}
// Optionally, get the fenced code block
string dslCode = extractCodeBlock(body, "dsl");
rv.SampleCode = dslCode;
return rv;
}
static string extractCodeBlock(string markdown, string language) {
string fence = "```" + language;
int start = markdown.IndexOf(fence);
if (start == -1) {
return string.Empty;
}
start += fence.Length;
int end = markdown.IndexOf("```", start);
if (end == -1) {
return string.Empty;
}
return markdown[start..end].Trim();
}
class NaturalLanguageStreamParams {
[StreamArgument(Position = 0)]
public string? Prompt { get; set; }
}
class PromptExample {
public string? Description { get; set; }
public string? NaturalLanguage { get; set; }
public string? SampleCode { get; set; }
}
}
@@ -4,4 +4,5 @@ public class NaturalLanguageStreamFactoryConfig {
public string? ApiKey { get; set; }
public string? MasterLearningPrompt { get; set; }
public string? ModelName { get; set; }
public List<string>? MasterLearningPromptExamples { get; set; }
}
+14
View File
@@ -95,6 +95,20 @@
<PublishOutDir>$(OutputPath)\publish\Configurations\$(MSBuildProjectName)</PublishOutDir>
<PublishOutFile>$(OutputPath)\publish\$(MSBuildProjectName).%(FileName)%(Extension)</PublishOutFile>
</ConfigurationProfile>
<BlueOrbPrompt>
<OutDir>$(OutputPath)\BlueOrbPrompts\$(MSBuildProjectName)</OutDir>
<OutFile>$(OutputPath)\$(MSBuildProjectName).%(FileName)%(Extension)</OutFile>
<PublishOutDir>$(OutputPath)\publish\BlueOrbPrompts\$(MSBuildProjectName)</PublishOutDir>
<PublishOutFile>$(OutputPath)\publish\$(MSBuildProjectName).%(FileName)%(Extension)</PublishOutFile>
</BlueOrbPrompt>
<BlueOrbPromptExample>
<OutDir>$(OutputPath)\BlueOrbPrompts\Examples\$(MSBuildProjectName)</OutDir>
<OutFile>$(OutputPath)\$(MSBuildProjectName).%(FileName)%(Extension)</OutFile>
<PublishOutDir>$(OutputPath)\publish\BlueOrbPrompts\Examples\$(MSBuildProjectName)</PublishOutDir>
<PublishOutFile>$(OutputPath)\publish\$(MSBuildProjectName).%(FileName)%(Extension)</PublishOutFile>
</BlueOrbPromptExample>
</ItemDefinitionGroup>
</Project>
+34
View File
@@ -65,6 +65,26 @@
<Copy SourceFiles="%(ConfigurationProfile.FullPath)" DestinationFolder="%(ConfigurationProfile.PublishOutDir)" />
</Target>
<Target
Name="CopyBlueOrbPrompts"
AfterTargets="Build"
Condition="'@(BlueOrbPrompt)'!=''"
Inputs="%(BlueOrbPrompt.Identity)"
Outputs="%(BlueOrbPrompt.OutDir)%(BlueOrbPrompt.Identity)">
<Copy SourceFiles="%(BlueOrbPrompt.FullPath)" DestinationFolder="%(BlueOrbPrompt.OutDir)" />
<Copy SourceFiles="%(BlueOrbPrompt.FullPath)" DestinationFiles="%(BlueOrbPrompt.OutFile)" />
</Target>
<Target
Name="CopyBlueOrbPromptExamples"
AfterTargets="Build"
Condition="'@(BlueOrbPromptExample)'!=''"
Inputs="%(BlueOrbPromptExample.Identity)"
Outputs="%(BlueOrbPromptExample.OutDir)%(BlueOrbPromptExample.Identity)">
<Copy SourceFiles="%(BlueOrbPromptExample.FullPath)" DestinationFolder="%(BlueOrbPromptExample.OutDir)" />
<Copy SourceFiles="%(BlueOrbPromptExample.FullPath)" DestinationFiles="%(BlueOrbPromptExample.OutFile)" />
</Target>
<!--
Run this after GetAssemblyVersion because it happens after the Xaml compiler
and exists in both xaml and non-xaml projects. One of the (many) xaml compile
@@ -192,6 +212,20 @@
<Delete Files="%(ConfigurationProfile.OutDir)\%(ConfigurationProfile.Identity)" />
</Target>
<Target
Name="CleanBlueOrbPrompts"
AfterTargets="Clean"
Condition="'@(BlueOrbPrompt)'!=''">
<Delete Files="%(BlueOrbPrompt.OutDir)\%(BlueOrbPrompt.Identity)" />
</Target>
<Target
Name="CleanBlueOrbPromptExamples"
AfterTargets="Clean"
Condition="'@(BlueOrbPromptExample)'!=''">
<Delete Files="%(BlueOrbPromptExample.OutDir)\%(BlueOrbPromptExample.Identity)" />
</Target>
<Target
Name="Test"
DependsOnTargets="Build"></Target>