refactor: Replace old NLStreamFactory w/ new BlueOrb one

Replace hardcoded file paths with proper config while we're at it
This commit is contained in:
John Colvin committed 2025-11-15 12:38:50 -05:00
1 parent 4ef2dbe4d7
commit eb28e04bf1
10 files changed
+1396 -151

No files matched your search

@@ -31,5 +31,7 @@ public static class DataStreamCoreServices {
.AddSingleton<IDataStreamFactory, TableStreamFactory>(sp => sp.GetRequiredService<TableStreamFactory>());
public static IServiceCollection AddCompositeDataStreamFactory(this IServiceCollection services) =>
services.AddSingleton<CompositeDataStreamFactory>();
services.AddSingleton<CompositeDataStreamFactory>()
// CompositeDataStreamFactory implements IHostedService to propagate itself to IRecursiveDataStreamFactory(s)
.AddHostedService(sp => sp.GetRequiredService<CompositeDataStreamFactory>());
}
File diff suppressed because it is too large. Load diff
@@ -14,17 +14,19 @@
[DataStreamUserRespresentation(false)]
public class BlueOrbNLStream : ViewStream {
private readonly string _nl_prompt;
readonly string masterLearningPrompt;
readonly IDataStreamFactory streamFactory;
ViewStream? _nl_to_dsl_stream;
ViewStream? _dsl_stream;
private IDictionary<string, object> _dp_metadata = new Dictionary<string, object>();
public BlueOrbNLStream(string nl_prompt, IDataStreamFactory streamFactory) {
public BlueOrbNLStream(string nl_prompt, string masterLearningPrompt, IDataStreamFactory streamFactory) {
_nl_prompt = nl_prompt;
this.masterLearningPrompt = masterLearningPrompt;
this.streamFactory = streamFactory;
}
protected override void myOnFirstSubscriber() {
_nl_to_dsl_stream = new BlueOrbNLToDSLStream(_nl_prompt);
_nl_to_dsl_stream = new BlueOrbNLToDSLStream(_nl_prompt, masterLearningPrompt);
_nl_to_dsl_stream.onChange += _handle_dsl_callback;
}
@@ -9,26 +9,9 @@
using System.Threading.Tasks;
using System.Runtime.InteropServices;
[DataStreamUserRespresentation("BlueOrbNLToDSLStream")]
[DataStreamUserRespresentation(false)]
public class BlueOrbNLToDSLStream : ViewStream {
public BlueOrbNLToDSLStream(string nl_prompt) {
string mlp_file_path = "C:\\Users\\arham\\Downloads\\" + "BlueOrbDSL_MLP_v1.9.md";
if (RuntimeInformation.IsOSPlatform(OSPlatform.Windows)) {
// Use current user's Downloads folder
string downloadsPath = Path.Combine(
Environment.GetFolderPath(Environment.SpecialFolder.UserProfile),
"Downloads"
);
mlp_file_path = Path.Combine(downloadsPath, "BlueOrbDSL_MLP_v1.9.md");
} else {
// On Linux, use ~/bd/
string home = Environment.GetEnvironmentVariable("HOME") ?? "/tmp";
string bldPath = Path.Combine(home, "BlueOrb_Data");
Directory.CreateDirectory(bldPath); // make sure it exists
mlp_file_path = Path.Combine(bldPath, "BlueOrbDSL_MLP_v1.9.md");
}
public BlueOrbNLToDSLStream(string nl_prompt, string mlp_file_path) {
string mlp_text = File.ReadAllText(mlp_file_path);
string system_prompt = $"You are a deterministic BlueOrbDSL generator. " +
$"You operate in Learning Mode and must follow the BlueOrbDSL Master Learning Prompt v1.9 strictly. " +
@@ -1,77 +0,0 @@
namespace bts.SilverPine;
using Spark.Core.Json;
using System.Collections.Generic;
using System.Text.Json;
using System.Text.Json.Serialization;
public class NLPromptConfig {
static NLPromptConfig? instance;
public static NLPromptConfig Instance =>
instance ?? throw new InvalidOperationException(
"SetGlobalConfig must be called to initialize config static singleton");
public List<FewShotPrompt> prompts { get; } = new(0);
public static void SetGlobalConfig(NLPromptConfig cfg) {
// This should only be called once
Log.isnull(instance);
instance = Log.notnull(cfg);
}
public static NLPromptConfig Default() => new();
// Represents a pair of a natural language query and its dsl
public class FewShotExample {
public string? natural_language { get; set; }
public string? dsl { get; set; }
}
public class FewShotPrompt {
public string? category { get; set; }
public string? description { get; set; }
public bool IsEnabled { get; set; } = true;
// MultiLineDescription is simply here for convenience. It is functionally redundant, as it
// serves the same purpose as 'description'. If 'description' is too long, you can break it
// up into an array of strings and assign it to MultiLineDescription instead.
public List<string>? MultiLineDescription { get; set; }
// the actual prompt given by the user/CME
public string? prompt { get; set; }
public List<FewShotExample> examples { get; } = new(0);
public override string ToString() {
var desc = description ?? string.Concat(MultiLineDescription.empty_if_null());
string rv = $"System: {desc}\n";
int ex_no = 0;
foreach (FewShotExample ex in examples ?? new(0)) {
rv += $"{++ex_no}. Natural Language: \"{ex.natural_language}\"" + "\n";
rv += $" DSL: {ex.dsl}\n";
}
return rv;
}
// Reads normal strings and arrays of lines
public class FewShotPromptLinesConverter : JsonConverter<string> {
public override string? Read(ref Utf8JsonReader reader, Type typeToConvert, JsonSerializerOptions options) {
if (reader.ReadString(out var str)) {
return str;
}
if (reader.ReadArray(out var array)) {
return string.Join('\n', array.Select(v => v?.AsString()));
}
throw new JsonException("Failed to parse FewShotPrompt lines");
}
public override void Write(Utf8JsonWriter writer, string value, JsonSerializerOptions options) =>
writer.WriteStringValue(value);
}
}
}
@@ -1,44 +0,0 @@
namespace bts.SilverPine;
using BCG;
using bts.BlueOrb;
using Bts.DataStreamCore;
using mbts.utils;
using System.Collections.Generic;
public class NLStreamFactory : IRecursiveDataStreamFactory {
IDataStreamFactory? compositeFactory;
public ViewStream? Create(string streamId, IEnumerable<(string? Name, int? Index, object? Value)> args) {
if (SupportedStreams.NaturalLanguage.IsMatch(streamId)) {
if (compositeFactory is null) {
Log.DataFlow.warn("Failed to create NLStream because 'compositeFactory' is null.");
return null;
}
if (args.SingleOrDefault().Value is string prompt) {
return new BlueOrbNLStream(prompt, compositeFactory);
}
}
return null;
}
public IEnumerable<IDataStreamMetadata> GetSupportedStreams() {
yield return SupportedStreams.NaturalLanguage;
}
public void OnCompositeFactory(IDataStreamFactory compositeFactory) =>
this.compositeFactory = compositeFactory;
static class SupportedStreams {
public static readonly DataStreamMetadata NaturalLanguage = new() {
Name = "nl",
AlternateNames = new() { "naturalLanguage" }
};
public static string NlName => ExceptionAssert.not_null(NaturalLanguage.Name);
}
}
@@ -16,14 +16,10 @@ public static class SharedStreamsServices {
.addStreamFactory<SpatialFuturePairsStreamFactory>()
.addStreamFactory<SpatialArbStatsStreamFactory>();
public static IServiceCollection AddNaturalLanguageStreams(this IServiceCollection services, IConfigurationSection configurationSection) {
services.Configure<NLPromptConfig>(configurationSection);
NLPromptConfig.SetGlobalConfig(configurationSection.Get<NLPromptConfig>() ?? NLPromptConfig.Default());
// TODO: integrate config into a NLStream provider (so it can provide config values
// instead of the ViewStream's getting them from a static singleton)
return services.AddSingleton<NLStreamFactory>()
.AddSingleton<IDataStreamFactory, NLStreamFactory>(sp => sp.GetRequiredService<NLStreamFactory>());
}
public static IServiceCollection AddNaturalLanguageStreams(this IServiceCollection services, IConfigurationSection configurationSection) =>
services.Configure<NaturalLanguageStreamFactoryConfig>(configurationSection)
.AddSingleton<NaturalLanguageStreamFactory>()
.AddSingleton<IDataStreamFactory, NaturalLanguageStreamFactory>(sp => sp.GetRequiredService<NaturalLanguageStreamFactory>());
static IServiceCollection addStreamFactory<T>(this IServiceCollection services) where T : class, IDataStreamFactory =>
services.AddSingleton<T>()
@@ -21,4 +21,8 @@
<PackageReference Include="System.Linq.Async" Version="6.0.1" />
</ItemGroup>
<ItemGroup>
<BlueOrbPrompt Include="BlueOrb\Prompts\BlueOrbDSL_MLP_v1.9.md" />
</ItemGroup>
</Project>
@@ -0,0 +1,39 @@
namespace Bts.SharedStreams.StreamFactories;
using BCG;
using bts.BlueOrb;
using Bts.DataStreamCore;
using Microsoft.Extensions.Logging;
using Microsoft.Extensions.Options;
public class NaturalLanguageStreamFactory : DataStreamFactory, IRecursiveDataStreamFactory {
IDataStreamFactory? compositeStreamFactory;
public NaturalLanguageStreamFactory(IOptions<NaturalLanguageStreamFactoryConfig> options, ILogger<NaturalLanguageStreamFactory> logger)
: this(options.Value, logger) { }
public NaturalLanguageStreamFactory(NaturalLanguageStreamFactoryConfig config, ILogger<NaturalLanguageStreamFactory> logger) {
this.Add<NaturalLanguageStreamParams>("nl", CreateNaturalLanguage, logger);
ViewStream? CreateNaturalLanguage(NaturalLanguageStreamParams streamParams) {
if (compositeStreamFactory is null) {
return null;
}
return new BlueOrbNLStream(
streamParams.Prompt ?? string.Empty,
config.MasterLearningPrompt ?? string.Empty,
compositeStreamFactory);
}
}
public void OnCompositeFactory(IDataStreamFactory compositeStreamFactory) =>
this.compositeStreamFactory = compositeStreamFactory;
class NaturalLanguageStreamParams {
[StreamArgument(Position = 0)]
public string? Prompt { get; set; }
}
}
@@ -0,0 +1,5 @@
namespace Bts.SharedStreams.StreamFactories;
public class NaturalLanguageStreamFactoryConfig {
public string? MasterLearningPrompt { get; set; }
}