This allows us to create loosely coupled services with automatic dependency resolution
129 lines
4.4 KiB
C#
129 lines
4.4 KiB
C#
namespace Bts.DataStream;
|
|
|
|
using BCG;
|
|
using BCG.Utilities.General;
|
|
|
|
using bts;
|
|
using bts.collections;
|
|
|
|
using Bts.DataStreamCore;
|
|
|
|
using Spark.Core.Disposables;
|
|
|
|
using System;
|
|
using System.Collections.Generic;
|
|
using System.Diagnostics;
|
|
using System.IO;
|
|
using System.Linq;
|
|
using System.Runtime.CompilerServices;
|
|
using System.Text;
|
|
|
|
public class PandasStream : ViewStream, IWorker {
|
|
readonly string inputFile = $"PandasStream_{Guid.NewGuid()}.csv";
|
|
readonly string outputFile = $"PandasStream_{Guid.NewGuid()}.csv";
|
|
readonly string pythonExe;
|
|
readonly CSVSaverStream inputCsvStream;
|
|
readonly Lazy<string> pythonCode;
|
|
IDictionary<int, IBlueRow>? lastRowMap = null;
|
|
int inputVersion = 0;
|
|
int outputVersion = 0;
|
|
DateTime lastFireTime = DateTime.MinValue;
|
|
TimeSpan fireThrottle = TimeSpan.FromSeconds(3);
|
|
IDisposable? dispose;
|
|
|
|
public PandasStream(ViewStream vs, string dataFrameExpression)
|
|
: this(vs, dataFrameExpression, null) { }
|
|
|
|
public PandasStream(ViewStream vs, string dataFrameExpression, string? pythonExe) {
|
|
pythonExe ??= Environment.GetEnvironmentVariable("PYTHON_EXE") ?? @"C:\Users\johnc\GoogleDrive\Repos\bts\spark\.venv\Scripts\python.exe";
|
|
this.pythonExe = pythonExe;
|
|
// This should fire an update when it writes to the output file
|
|
inputCsvStream = new CSVSaverStream(vs, inputFile, true, int.MaxValue, true);
|
|
pythonCode = new(getPythonCode);
|
|
|
|
string getPythonCode() {
|
|
var imports =
|
|
@"import numpy as np
|
|
import pandas as pd
|
|
import sys
|
|
";
|
|
var bd = new StringBuilder(imports);
|
|
bd.AppendLine($"df=pd.read_csv('{inputFile}')");
|
|
bd.AppendLine(dataFrameExpression);
|
|
bd.AppendLine($"df.to_csv('{outputFile}', index=False)");
|
|
return bd.ToString();
|
|
}
|
|
}
|
|
|
|
protected override void myOnFirstSubscriber() {
|
|
base.myOnFirstSubscriber();
|
|
inputCsvStream.onChange += onInputChanged;
|
|
this.dispose = Disposable.Create(dispose);
|
|
|
|
void onInputChanged(object newValue) =>
|
|
inputVersion++;
|
|
|
|
void dispose() {
|
|
inputCsvStream.onChange -= onInputChanged;
|
|
deleteFile(inputFile);
|
|
deleteFile(outputFile);
|
|
}
|
|
|
|
void deleteFile(string file, [CallerArgumentExpression(nameof(file))] string name = "") {
|
|
if (!File.Exists(file)) {
|
|
return;
|
|
}
|
|
try {
|
|
File.Delete(file);
|
|
} catch (Exception ex) {
|
|
Log.Gen.error($"PandasStream. Error deleting {name} file '{file}': {ex}");
|
|
}
|
|
}
|
|
}
|
|
|
|
protected override void myOnLastSubscriber() {
|
|
base.myOnLastSubscriber();
|
|
Disposable.dispose(ref dispose);
|
|
}
|
|
|
|
static void executePythonCode(string pythonCode, string? pythonExe = null) {
|
|
pythonExe ??= "python";
|
|
var start = new ProcessStartInfo {
|
|
FileName = pythonExe,
|
|
Arguments = "-c \"" + pythonCode.Replace("\"", "\\\"") + "\"",
|
|
UseShellExecute = false,
|
|
RedirectStandardOutput = true
|
|
};
|
|
using var process = Process.Start(start)
|
|
?? throw new InvalidOperationException("Failed to launch python");
|
|
process.WaitForExit();
|
|
}
|
|
|
|
void IWorker.doWork() {
|
|
if (outputVersion == inputVersion) {
|
|
return;
|
|
}
|
|
var now = DateTime.Now;
|
|
var elapsedSinceLastFire = now - lastFireTime;
|
|
if (elapsedSinceLastFire < fireThrottle) {
|
|
return;
|
|
}
|
|
outputVersion = inputVersion;
|
|
executePythonCode(pythonCode.Value, pythonExe);
|
|
var rows = BlueDataUtility.createRowsFromCSVFileWithoutTypes(outputFile, out var schema);
|
|
var rowMap = rows as IDictionary<int, IBlueRow> ?? rows.Cast<IBlueRow>().ToDictionary(r => r.Id());
|
|
var comp = BlueRowEqualityComparer.Instance;
|
|
var deletedRows = new BlueTable(schema);
|
|
var addedRows = new BlueTable(schema);
|
|
if (lastRowMap is not null) {
|
|
deletedRows.SetItems(lastRowMap.Where(oldRow =>
|
|
!(rowMap.TryGetValue(oldRow.Key, out var newRow) && comp.Equals(newRow, oldRow.Value))));
|
|
addedRows.SetItems(rowMap.Where(newRow =>
|
|
!(lastRowMap.TryGetValue(newRow.Key, out var oldRow) && comp.Equals(oldRow, newRow.Value))));
|
|
}
|
|
lastRowMap = rowMap;
|
|
lastFireTime = DateTime.Now;
|
|
myFireEvent(new DataPacket(rows, addedRows, deletedRows));
|
|
}
|
|
}
|