improve: Generate MLP at runtime with list of example files
Adds prompt file list to NaturalLanguageStreamFactoryConfig
This commit is contained in:
1 parent
0f9e515d01
commit
94e9c8c3f2
5 files changed
+147
No files matched your search
@@ -20,10 +20,13 @@
|
||||
<PackageReference Include="OpenAI" Version="2.0.0" />
|
||||
<PackageReference Include="System.Linq.Async" Version="6.0.1" />
|
||||
<PackageReference Include="Parquet.Net" Version="5.2.0" />
|
||||
<PackageReference Include="YamlDotNet" Version="16.3.0" />
|
||||
</ItemGroup>
|
||||
|
||||
<ItemGroup>
|
||||
<BlueOrbPrompt Include="BlueOrb\Prompts\BlueOrbDSL_MLP_v1.9.md" />
|
||||
<BlueOrbPrompt Include="BlueOrb\Prompts\BrandonChenMLP.md" />
|
||||
<BlueOrbPromptExample Include="BlueOrb\Prompts\Examples\**\*.md" />
|
||||
</ItemGroup>
|
||||
|
||||
</Project>
|
||||
@@ -2,13 +2,19 @@ namespace Bts.SharedStreams.StreamFactories;
|
||||
|
||||
using BCG;
|
||||
|
||||
using bts;
|
||||
using bts.BlueOrb;
|
||||
using bts.collections;
|
||||
|
||||
using Bts.DataStreamCore;
|
||||
|
||||
using Microsoft.Extensions.Logging;
|
||||
using Microsoft.Extensions.Options;
|
||||
|
||||
using System.Text;
|
||||
|
||||
using YamlDotNet.RepresentationModel;
|
||||
|
||||
public class NaturalLanguageStreamFactory : DataStreamFactory, IRecursiveDataStreamFactory {
|
||||
IDataStreamFactory? compositeStreamFactory;
|
||||
|
||||
@@ -16,6 +22,39 @@ public class NaturalLanguageStreamFactory : DataStreamFactory, IRecursiveDataStr
|
||||
: this(options.Value, viewStreamDatabase, logger) { }
|
||||
|
||||
public NaturalLanguageStreamFactory(NaturalLanguageStreamFactoryConfig config, ViewStreamDatabase viewStreamDatabase, ILogger<NaturalLanguageStreamFactory> logger) {
|
||||
string prompt = config.MasterLearningPrompt ?? string.Empty;
|
||||
if (config.MasterLearningPromptExamples?.Count > 0) {
|
||||
var dir = Path.GetDirectoryName(prompt);
|
||||
Log.notnull(dir);
|
||||
var promptText = File.ReadAllLines(prompt);
|
||||
prompt = Path.Combine(dir, Path.GetFileNameWithoutExtension(config.MasterLearningPrompt) + "_generated.md");
|
||||
var examplesBd = new StringBuilder();
|
||||
int exNum = 0;
|
||||
foreach (var e in config.MasterLearningPromptExamples.Select(loadExample).ExceptNull()) {
|
||||
exNum++;
|
||||
examplesBd.AppendLine($"### Example G{exNum} - {e.Description}");
|
||||
examplesBd.AppendLine();
|
||||
examplesBd.AppendLine("**NL Query:**");
|
||||
examplesBd.AppendLine(e.NaturalLanguage);
|
||||
examplesBd.AppendLine();
|
||||
examplesBd.AppendLine("```");
|
||||
examplesBd.AppendLine(e.SampleCode);
|
||||
examplesBd.AppendLine("```");
|
||||
examplesBd.AppendLine();
|
||||
examplesBd.AppendLine("---");
|
||||
examplesBd.AppendLine();
|
||||
}
|
||||
File.WriteAllLines(prompt, promptText);
|
||||
using var fw = File.OpenWrite(prompt);
|
||||
using var sw = new StreamWriter(fw);
|
||||
foreach (var line in promptText) {
|
||||
if (string.Equals(line, "{EXAMPLES}", StringComparison.CurrentCultureIgnoreCase)) {
|
||||
sw.WriteLine(examplesBd.ToString());
|
||||
} else {
|
||||
sw.WriteLine(line);
|
||||
}
|
||||
}
|
||||
}
|
||||
this.Add<NaturalLanguageStreamParams>("nl", CreateNaturalLanguage, logger);
|
||||
|
||||
ViewStream? CreateNaturalLanguage(NaturalLanguageStreamParams streamParams) {
|
||||
@@ -35,8 +74,64 @@ public class NaturalLanguageStreamFactory : DataStreamFactory, IRecursiveDataStr
|
||||
public void OnCompositeFactory(IDataStreamFactory compositeStreamFactory) =>
|
||||
this.compositeStreamFactory = compositeStreamFactory;
|
||||
|
||||
static PromptExample? loadExample(string file) {
|
||||
string text = File.ReadAllText(file);
|
||||
|
||||
// Extract YAML frontmatter
|
||||
var parts = text.Split(new string[] { "---" }, StringSplitOptions.None);
|
||||
if (parts.Length < 3) {
|
||||
Console.WriteLine("No frontmatter found.");
|
||||
return null;
|
||||
}
|
||||
|
||||
string yaml = parts[1]; // frontmatter is always between the first two '---'
|
||||
string body = string.Join("---", parts.Skip(2)); // the rest is the markdown body
|
||||
|
||||
// Parse YAML
|
||||
var yamlStream = new YamlStream();
|
||||
yamlStream.Load(new StringReader(yaml));
|
||||
var root = (YamlMappingNode)yamlStream.Documents[0].RootNode;
|
||||
var rv = new PromptExample();
|
||||
|
||||
// Get "desc"
|
||||
if (root.Children.TryGetValue(new YamlScalarNode("desc"), out var descNode)) {
|
||||
rv.Description = descNode.ToString();
|
||||
}
|
||||
|
||||
// Get "desc"
|
||||
if (root.Children.TryGetValue(new YamlScalarNode("nl"), out var nlNode)) {
|
||||
rv.NaturalLanguage = nlNode.ToString();
|
||||
}
|
||||
// Optionally, get the fenced code block
|
||||
string dslCode = extractCodeBlock(body, "dsl");
|
||||
rv.SampleCode = dslCode;
|
||||
return rv;
|
||||
}
|
||||
|
||||
static string extractCodeBlock(string markdown, string language) {
|
||||
string fence = "```" + language;
|
||||
int start = markdown.IndexOf(fence);
|
||||
if (start == -1) {
|
||||
return string.Empty;
|
||||
}
|
||||
|
||||
start += fence.Length;
|
||||
int end = markdown.IndexOf("```", start);
|
||||
if (end == -1) {
|
||||
return string.Empty;
|
||||
}
|
||||
|
||||
return markdown[start..end].Trim();
|
||||
}
|
||||
|
||||
class NaturalLanguageStreamParams {
|
||||
[StreamArgument(Position = 0)]
|
||||
public string? Prompt { get; set; }
|
||||
}
|
||||
|
||||
class PromptExample {
|
||||
public string? Description { get; set; }
|
||||
public string? NaturalLanguage { get; set; }
|
||||
public string? SampleCode { get; set; }
|
||||
}
|
||||
}
|
||||
+1
@@ -4,4 +4,5 @@ public class NaturalLanguageStreamFactoryConfig {
|
||||
public string? ApiKey { get; set; }
|
||||
public string? MasterLearningPrompt { get; set; }
|
||||
public string? ModelName { get; set; }
|
||||
public List<string>? MasterLearningPromptExamples { get; set; }
|
||||
}
|
||||
@@ -95,6 +95,20 @@
|
||||
<PublishOutDir>$(OutputPath)\publish\Configurations\$(MSBuildProjectName)</PublishOutDir>
|
||||
<PublishOutFile>$(OutputPath)\publish\$(MSBuildProjectName).%(FileName)%(Extension)</PublishOutFile>
|
||||
</ConfigurationProfile>
|
||||
|
||||
<BlueOrbPrompt>
|
||||
<OutDir>$(OutputPath)\BlueOrbPrompts\$(MSBuildProjectName)</OutDir>
|
||||
<OutFile>$(OutputPath)\$(MSBuildProjectName).%(FileName)%(Extension)</OutFile>
|
||||
<PublishOutDir>$(OutputPath)\publish\BlueOrbPrompts\$(MSBuildProjectName)</PublishOutDir>
|
||||
<PublishOutFile>$(OutputPath)\publish\$(MSBuildProjectName).%(FileName)%(Extension)</PublishOutFile>
|
||||
</BlueOrbPrompt>
|
||||
|
||||
<BlueOrbPromptExample>
|
||||
<OutDir>$(OutputPath)\BlueOrbPrompts\Examples\$(MSBuildProjectName)</OutDir>
|
||||
<OutFile>$(OutputPath)\$(MSBuildProjectName).%(FileName)%(Extension)</OutFile>
|
||||
<PublishOutDir>$(OutputPath)\publish\BlueOrbPrompts\Examples\$(MSBuildProjectName)</PublishOutDir>
|
||||
<PublishOutFile>$(OutputPath)\publish\$(MSBuildProjectName).%(FileName)%(Extension)</PublishOutFile>
|
||||
</BlueOrbPromptExample>
|
||||
</ItemDefinitionGroup>
|
||||
|
||||
</Project>
|
||||
@@ -65,6 +65,26 @@
|
||||
<Copy SourceFiles="%(ConfigurationProfile.FullPath)" DestinationFolder="%(ConfigurationProfile.PublishOutDir)" />
|
||||
</Target>
|
||||
|
||||
<Target
|
||||
Name="CopyBlueOrbPrompts"
|
||||
AfterTargets="Build"
|
||||
Condition="'@(BlueOrbPrompt)'!=''"
|
||||
Inputs="%(BlueOrbPrompt.Identity)"
|
||||
Outputs="%(BlueOrbPrompt.OutDir)%(BlueOrbPrompt.Identity)">
|
||||
<Copy SourceFiles="%(BlueOrbPrompt.FullPath)" DestinationFolder="%(BlueOrbPrompt.OutDir)" />
|
||||
<Copy SourceFiles="%(BlueOrbPrompt.FullPath)" DestinationFiles="%(BlueOrbPrompt.OutFile)" />
|
||||
</Target>
|
||||
|
||||
<Target
|
||||
Name="CopyBlueOrbPromptExamples"
|
||||
AfterTargets="Build"
|
||||
Condition="'@(BlueOrbPromptExample)'!=''"
|
||||
Inputs="%(BlueOrbPromptExample.Identity)"
|
||||
Outputs="%(BlueOrbPromptExample.OutDir)%(BlueOrbPromptExample.Identity)">
|
||||
<Copy SourceFiles="%(BlueOrbPromptExample.FullPath)" DestinationFolder="%(BlueOrbPromptExample.OutDir)" />
|
||||
<Copy SourceFiles="%(BlueOrbPromptExample.FullPath)" DestinationFiles="%(BlueOrbPromptExample.OutFile)" />
|
||||
</Target>
|
||||
|
||||
<!--
|
||||
Run this after GetAssemblyVersion because it happens after the Xaml compiler
|
||||
and exists in both xaml and non-xaml projects. One of the (many) xaml compile
|
||||
@@ -192,6 +212,20 @@
|
||||
<Delete Files="%(ConfigurationProfile.OutDir)\%(ConfigurationProfile.Identity)" />
|
||||
</Target>
|
||||
|
||||
<Target
|
||||
Name="CleanBlueOrbPrompts"
|
||||
AfterTargets="Clean"
|
||||
Condition="'@(BlueOrbPrompt)'!=''">
|
||||
<Delete Files="%(BlueOrbPrompt.OutDir)\%(BlueOrbPrompt.Identity)" />
|
||||
</Target>
|
||||
|
||||
<Target
|
||||
Name="CleanBlueOrbPromptExamples"
|
||||
AfterTargets="Clean"
|
||||
Condition="'@(BlueOrbPromptExample)'!=''">
|
||||
<Delete Files="%(BlueOrbPromptExample.OutDir)\%(BlueOrbPromptExample.Identity)" />
|
||||
</Target>
|
||||
|
||||
<Target
|
||||
Name="Test"
|
||||
DependsOnTargets="Build"></Target>
|
||||
|
||||
Reference in new issue
Block a user