Files
spark/src/DataStreamSuite/DataStream/Misc/PandasStream.cs
T
John Colvin a05c7410dc refactor: Use DependencyInjection paradigm in DataStreamSuite
This allows us to create loosely coupled services with automatic
dependency resolution
2024-07-02 11:30:14 -04:00

129 lines
4.4 KiB
C#

namespace Bts.DataStream;
using BCG;
using BCG.Utilities.General;
using bts;
using bts.collections;
using Bts.DataStreamCore;
using Spark.Core.Disposables;
using System;
using System.Collections.Generic;
using System.Diagnostics;
using System.IO;
using System.Linq;
using System.Runtime.CompilerServices;
using System.Text;
public class PandasStream : ViewStream, IWorker {
readonly string inputFile = $"PandasStream_{Guid.NewGuid()}.csv";
readonly string outputFile = $"PandasStream_{Guid.NewGuid()}.csv";
readonly string pythonExe;
readonly CSVSaverStream inputCsvStream;
readonly Lazy<string> pythonCode;
IDictionary<int, IBlueRow>? lastRowMap = null;
int inputVersion = 0;
int outputVersion = 0;
DateTime lastFireTime = DateTime.MinValue;
TimeSpan fireThrottle = TimeSpan.FromSeconds(3);
IDisposable? dispose;
public PandasStream(ViewStream vs, string dataFrameExpression)
: this(vs, dataFrameExpression, null) { }
public PandasStream(ViewStream vs, string dataFrameExpression, string? pythonExe) {
pythonExe ??= Environment.GetEnvironmentVariable("PYTHON_EXE") ?? @"C:\Users\johnc\GoogleDrive\Repos\bts\spark\.venv\Scripts\python.exe";
this.pythonExe = pythonExe;
// This should fire an update when it writes to the output file
inputCsvStream = new CSVSaverStream(vs, inputFile, true, int.MaxValue, true);
pythonCode = new(getPythonCode);
string getPythonCode() {
var imports =
@"import numpy as np
import pandas as pd
import sys
";
var bd = new StringBuilder(imports);
bd.AppendLine($"df=pd.read_csv('{inputFile}')");
bd.AppendLine(dataFrameExpression);
bd.AppendLine($"df.to_csv('{outputFile}', index=False)");
return bd.ToString();
}
}
protected override void myOnFirstSubscriber() {
base.myOnFirstSubscriber();
inputCsvStream.onChange += onInputChanged;
this.dispose = Disposable.Create(dispose);
void onInputChanged(object newValue) =>
inputVersion++;
void dispose() {
inputCsvStream.onChange -= onInputChanged;
deleteFile(inputFile);
deleteFile(outputFile);
}
void deleteFile(string file, [CallerArgumentExpression(nameof(file))] string name = "") {
if (!File.Exists(file)) {
return;
}
try {
File.Delete(file);
} catch (Exception ex) {
Log.Gen.error($"PandasStream. Error deleting {name} file '{file}': {ex}");
}
}
}
protected override void myOnLastSubscriber() {
base.myOnLastSubscriber();
Disposable.dispose(ref dispose);
}
static void executePythonCode(string pythonCode, string? pythonExe = null) {
pythonExe ??= "python";
var start = new ProcessStartInfo {
FileName = pythonExe,
Arguments = "-c \"" + pythonCode.Replace("\"", "\\\"") + "\"",
UseShellExecute = false,
RedirectStandardOutput = true
};
using var process = Process.Start(start)
?? throw new InvalidOperationException("Failed to launch python");
process.WaitForExit();
}
void IWorker.doWork() {
if (outputVersion == inputVersion) {
return;
}
var now = DateTime.Now;
var elapsedSinceLastFire = now - lastFireTime;
if (elapsedSinceLastFire < fireThrottle) {
return;
}
outputVersion = inputVersion;
executePythonCode(pythonCode.Value, pythonExe);
var rows = BlueDataUtility.createRowsFromCSVFileWithoutTypes(outputFile, out var schema);
var rowMap = rows as IDictionary<int, IBlueRow> ?? rows.Cast<IBlueRow>().ToDictionary(r => r.Id());
var comp = BlueRowEqualityComparer.Instance;
var deletedRows = new BlueTable(schema);
var addedRows = new BlueTable(schema);
if (lastRowMap is not null) {
deletedRows.SetItems(lastRowMap.Where(oldRow =>
!(rowMap.TryGetValue(oldRow.Key, out var newRow) && comp.Equals(newRow, oldRow.Value))));
addedRows.SetItems(rowMap.Where(newRow =>
!(lastRowMap.TryGetValue(newRow.Key, out var oldRow) && comp.Equals(oldRow, newRow.Value))));
}
lastRowMap = rowMap;
lastFireTime = DateTime.Now;
myFireEvent(new DataPacket(rows, addedRows, deletedRows));
}
}