diff --git a/src/DataStreamSuite/DataStream/BlueOrb/PythonDataframeStream.cs b/src/DataStreamSuite/DataStream/BlueOrb/PythonDataframeStream.cs index 0c1ac938c..a8f9625ca 100644 --- a/src/DataStreamSuite/DataStream/BlueOrb/PythonDataframeStream.cs +++ b/src/DataStreamSuite/DataStream/BlueOrb/PythonDataframeStream.cs @@ -9,11 +9,13 @@ private ViewStream _parent_str; private readonly string _in_csv_file_name; private readonly string _out_csv_filename; - public PythonDataframeStream(ViewStream parent_str) { + private readonly string _transform_function; + public PythonDataframeStream(ViewStream parent_str, string transform_function) { var guid = System.Guid.NewGuid(); - _in_csv_file_name = $"pdfs_in_{guid}"; + _in_csv_file_name = $"pdfs_in_{guid}.csv"; + _out_csv_filename = $"pdfs_out_{guid}.csv"; _parent_str = new CSVSaverStream(parent_str, _in_csv_file_name); - //_simple_test(); + _transform_function = transform_function; } protected override void myOnFirstSubscriber() { @@ -24,7 +26,8 @@ _parent_str.onChange -= _parent_str_onchange; } - private void _parent_str_onchange(object obj) { + private void _parent_str_onchange(object dp_obj) { + Log.assert(dp_obj != null); // the file _in_csv_file_name is now populated. // call py_func_wrapper_main and pass this file name as well // as the output file name @@ -33,25 +36,49 @@ // writes it into the _out_csv_filename file. // now this function should read _out_csv_filename into a dp // and fire it + _compute_and_write(); + ViewStream rs = new SchemaGuesserStream(new CSVReaderStream(_out_csv_filename)); + IBlueRowCollection nrows = null; + rs.onChange += (dp_obj) => { + nrows = (dp_obj as DataPacket).newRows; + }; + var dp = BlueDataUtility.build_dp(nrows, myLastValue as DataPacket); + myFireEvent(dp); } - private void _compute() { + private void _compute_and_write() { + File.WriteAllText("user_code.py", _transform_function); + // wrapper function string py_func_wrapper_main = @" import sys -from user_code import test +from user_code import transform +import pandas as pd if __name__ == '__main__': - inp_file_name = sys.argv[1] + input_file_name = sys.argv[1] output_file_name = sys.argv[2] -# 1. load inp_file_name into df +# 1. load input_file_name into df # 2. do some test calc # 3. write the output of 2 into output_file_name - rv = test(arg, out_path) - rv = rv + 12 - with open(out_path, 'w') as f: - f.write(str(rv))"; + df = pd.read_csv(input_file_name) + df_out = transform(df) + df_out.to_csv(output_file_name, index=False)"; + + + File.WriteAllText("runner.py", py_func_wrapper_main); + + ProcessStartInfo psi = new ProcessStartInfo { + FileName = "python", + Arguments = $"runner.py {_in_csv_file_name} {_out_csv_filename}", + RedirectStandardOutput = false, + UseShellExecute = false + }; + var proc = Process.Start(psi); + proc.WaitForExit(); + } + #region Unused test code private void _simple_test() { // actual function that needs to be called //////////////////////// @@ -95,5 +122,6 @@ if __name__ == '__main__': Console.WriteLine(result); // Should print "6" } + #endregion } }