using System; using System.Collections.Generic; using System.Linq; using System.Text; using System.IO; using System.Runtime.Serialization.Formatters.Binary; using BCG.Utilities.General; using bts.collections; using System.Reflection.Metadata; namespace BCG { public class PlaybackStream : ViewStream, IWorker { private readonly PlaybackType _type; private BlueTable _current_table; private IClientDeserializer _deserilializer = new ClientUniversalDeserializer(); // these are the packets already downloaded from the current file private List _packets = new List(); //private IEnumerable _file_streams = new List(); private IEnumerable _file_names; private Tuple _current_fs = new Tuple(-1, null); private DateTime _playback_t0, _playback_t1; private bool _all_files_read = false; SnapshotData _snapshot_data = null; // only initialized when in snapshot mode public PlaybackStream(string datetime_start, string datetime_end, string directory_name, string token) { _type = PlaybackType.Streaming; _playback_t0 = PersistorStream.parse_datetime(datetime_start); _playback_t1 = PersistorStream.parse_datetime(datetime_end); string token_directory_name = $"{directory_name}\\{token}"; bts.Log.Gen.info($"PlaybackStream t0: {_playback_t0} t1: {_playback_t1} dir: {token_directory_name}"); List files_in_range = new List(); if (Directory.Exists(token_directory_name)) { _file_names = Directory.GetFiles(token_directory_name).Where(f => { DateTime file_time = PersistorStream.parse_datetime_from_filename(f); return file_time > _playback_t0 && file_time < _playback_t1; }); } } protected override void myOnFirstSubscriber() { FileStream fs = new FileStream(_snapshot_data.snapshot_file, FileMode.Open, FileAccess.Read); _snapshot_data.packets = StreamPersistorUtils.load_packets_from_file(fs); PlaybackPacket snapshot_packet = StreamPersistorUtils.find_packet_at_time(_snapshot_data.packets, _snapshot_data.snapshot_time); if (snapshot_packet != null) { foreach (PlaybackPacket p in _snapshot_data.packets) { byte[] ops; Array row_array; _deserilializer.deserialize(new Stream[] { new MemoryStream(p.data) }, out ops, out row_array); for (int i = 0; i < ops.Length; i++) { IBlueRowCollection[] rows_args = row_array.GetValue(i) as IBlueRowCollection[]; byte op = ops[i]; if (op == StreamOpCode.create) { IBlueRowCollection full_table = rows_args[0]; _current_table = new BlueTable(full_table); } else if (op == StreamOpCode.multipleRemoveAdd) { //IBlueRowCollection[] del_added = rows_args.GetValue(0) as IBlueRowCollection[]; IBlueRowCollection dels = rows_args.GetValue(0) as IBlueRowCollection; IBlueRowCollection adds = rows_args.GetValue(1) as IBlueRowCollection; BlueDataUtility.absorbChanges(_current_table, dels, adds); } else { bts.Log.assert(false, $"Oprator: {op} is not expected"); } if (p.idx == snapshot_packet.idx) { myFireEvent(new DataPacket(_current_table, _current_table, new BlueTable(_current_table.schema))); break; } } } } } private void _load_packets_from_current_file() { int curr_idx = _current_fs.Item1; FileStream curr_fs = _current_fs.Item2; _packets = new List(); _packets.AddRange(StreamPersistorUtils.load_packets_from_file(curr_fs)); } private bool current_file_all_replayed { get { bool rv = true; _packets.ForEach(p => rv = rv && p.replayed); return rv; } } protected override void myOnLastSubscriber() { var fs = _current_fs.Item2; if (fs != null) { fs.Close(); } ViewStreamDatabase.instance.removeWorker(this); } // public event PlaybackStreamProgress onProgress; // public event Action onFileExhausted; private PlaybackPacket _locate_packet_to_replay() { foreach (var p in _packets) { // returns the first non replayed packet if (!p.replayed) return p; } return null; } private PlaybackPacket _get_previous_packet(PlaybackPacket p) { if (p.idx == 0) return null; return _packets[p.idx - 1]; } #region IWorker Members public void doWork() { if (_all_files_read) return; if (current_file_all_replayed) { _load_next_file(); } PlaybackPacket curr_packet = _locate_packet_to_replay(); if (curr_packet.idx == 0) { // first packet in the file, fire it rightaway byte[] ops; Array row_array; // what a stupid interface! _deserilializer.deserialize(new Stream[] { new MemoryStream(curr_packet.data) }, out ops, out row_array); _fire_packet(ops, row_array); curr_packet.replay_timestamp = PersistorStream.epoch_now(); } else { PlaybackPacket prev_packet = _get_previous_packet(curr_packet); // this is the time we need to fire (replay) this packet long time_to_fire = prev_packet.replay_timestamp + curr_packet.offset_from_previous; long now_epoch = PersistorStream.epoch_now(); // fire when we are past the if (now_epoch >= time_to_fire) { byte[] ops; Array row_array; _deserilializer.deserialize(new Stream[] { new MemoryStream(curr_packet.data) }, out ops, out row_array); _fire_packet(ops, row_array); curr_packet.replay_timestamp = PersistorStream.epoch_now(); } } } private void _fire_packet(byte[] ops, Array row_array) { for (int i = 0; i < ops.Length; i++) { IBlueRowCollection[] rows_args = row_array.GetValue(i) as IBlueRowCollection[]; byte op = ops[i]; if (op == StreamOpCode.create) { IBlueRowCollection full_table = rows_args[0]; _current_table = new BlueTable(full_table); //_current_table.addRows(full_table); myFireEvent(new DataPacket(full_table, full_table, myLastValue == null ? new BlueTable(full_table.schema) : (myLastValue as DataPacket).newRows)); } else if (op == StreamOpCode.multipleRemoveAdd) { //IBlueRowCollection[] del_added = rows_args.GetValue(0) as IBlueRowCollection[]; IBlueRowCollection dels = rows_args.GetValue(0) as IBlueRowCollection; IBlueRowCollection adds = rows_args.GetValue(1) as IBlueRowCollection; BlueDataUtility.absorbChanges(_current_table, dels, adds); myFireEvent(new DataPacket(_current_table, adds, dels)); } else { bts.Log.assert(false, $"Oprator: {op} is not expected"); } } } private void _load_next_file() { var new_idx = _current_fs.Item1 + 1; if (new_idx >= _file_names.Count()) { _all_files_read = true; bts.Log.Gen.info("All files read"); return; } var prev_fs = _current_fs.Item2; if (prev_fs != null) prev_fs.Close(); var new_filename = _file_names.ElementAt(new_idx); bts.Log.Gen.info("PlaybackStream_load_next_file new_filename=: {new_filename}"); var new_fs = new FileStream(new_filename, FileMode.Open, FileAccess.Read); _current_fs = new Tuple(new_idx, new_fs); _load_packets_from_current_file(); } #endregion } }