diff --git a/src/DataStreamSuite/DataStream/BlueOrb/BlueAddIndexColumnTable.cs b/src/DataStreamSuite/DataStream/BlueOrb/BlueAddIndexColumnTable.cs new file mode 100644 index 000000000..ae0c11380 --- /dev/null +++ b/src/DataStreamSuite/DataStream/BlueOrb/BlueAddIndexColumnTable.cs @@ -0,0 +1,118 @@ +namespace Bts.DataStream; + +using BCG; + +using bts.collections; +using bts.utils; + +using Bts.DataStreamCore; + +using Spark.Core.Collections; + +using System; +using System.Collections; +using System.Collections.Generic; +using System.Linq; + +public class BlueAddIndexColumnTable : IBlueRowCollection { + readonly IBlueTableSchema schema; + readonly IList[] columnValues; + readonly int count; + + BlueAddIndexColumnTable(IBlueTableSchema schema, IList[] columnValues, int? count = null) { + this.schema = schema; + this.columnValues = columnValues; + this.count = count ?? columnValues[0].Count; + } + + IBlueTableSchema IBlueRowCollection.schema => schema; + + public int Count => count; + + public IBlueRow this[int id] => + id > 0 && id <= count + ? GetRow(id - 1) + : throw new KeyNotFoundException(); + + public static BlueAddIndexColumnTable Create(string tableName, ColumnInfo[] columnInfos, IList[] columnValues) { + if (columnInfos.Any(c => c.colName == "id")) { + throw new ArgumentException("'id' column already exists", nameof(columnInfos)); + } + if (columnInfos.Length != columnValues.Length) { + throw new ArgumentException("columnValues length differs from columnInfos length", nameof(columnValues)); + } + int count = columnValues.FirstOrDefault()?.Count ?? 0; + var origCols = columnInfos; + var origColValues = columnValues; + columnInfos = new ColumnInfo[origCols.Length + 1]; + columnInfos[0] = new ColumnInfo("id", 0, typeof(int)); + columnValues = new IList[origCols.Length + 1]; + columnValues[0] = new StepList(1, 1, count).WrapSelector(id => (object?)id, null); + for (int i = 1; i < columnInfos.Length; i++) { + columnInfos[i] = new ColumnInfo(origCols[i - 1].colName, i, origCols[i - 1].colType); + columnValues[i] = origColValues[i - 1]; + } + var schema = new BlueTableSchema(columnInfos, tableName); + return new(schema, columnValues, count); + } + + public static BlueAddIndexColumnTable Create(IBlueTableSchema schema, IList[] columnValues) { + int count = columnValues.FirstOrDefault()?.Count ?? 0; + int? idColIdx = schema.TryResolveIndices("id"); + var idColValues = new StepList(1, 1, count).WrapSelector(id => (object?)id, null); + var columnInfos = schema.columnInfo; + if (idColIdx.HasValue) { + // Replace existing id column. columnValues may or may not contain the old ids (if its + // length is one less column than that of columnInfos, it doesn't contain them). + switch (columnValues.Length - schema.columnInfo.Length) { + case 0: + columnValues[idColIdx.Value] = idColValues; + break; + case -1: + columnValues = ArrayHelper.InsertExact(columnValues, idColIdx.Value, true, idColValues); + break; + default: + throw new ArgumentException("ColumnInfos length disagrees with columnValues length"); + } + } else { + if (columnValues.Length != columnInfos.Length) { + throw new ArgumentException("ColumnInfos length disagrees with columnValues length"); + } + var origColInfos = columnInfos; + columnInfos = new ColumnInfo[columnInfos.Length + 1]; + columnInfos[0] = new("id", 0, typeof(int)); + for (int i = 1; i < columnInfos.Length; i++) { + var c = origColInfos[i - 1]; + columnInfos[i] = new(c.colName, c.colNo + 1, c.colType); + } + columnValues = ArrayHelper.InsertExact(columnValues, 0, true, idColValues); + } + return new(new BlueTableSchema(schema.tableName, columnInfos), columnValues); + } + + public bool containsID(int id) => + id > 0 && id <= count; + + public IList GetColumn(int index) => + columnValues[index]; + + public IBlueRow GetRow(int index) => + new BlueRowColumnWrapper(schema, index, columnValues); + + public int IndexOf(IBlueRow item) { + int id = item.Id(); + return id > 0 && id <= count && BlueRowEqualityComparer.Equals(GetRow(id - 1), item) + ? id - 1 + : -1; + } + + public IEnumerator GetEnumerator() { + int count = Count; + for (int i = 0; i < count; i++) { + yield return GetRow(i); + } + } + + IEnumerator IEnumerable.GetEnumerator() => + GetEnumerator(); +} diff --git a/src/DataStreamSuite/DataStream/BlueOrb/BlueIndexedColumnTable.cs b/src/DataStreamSuite/DataStream/BlueOrb/BlueIndexedColumnTable.cs new file mode 100644 index 000000000..6b43a4bc2 --- /dev/null +++ b/src/DataStreamSuite/DataStream/BlueOrb/BlueIndexedColumnTable.cs @@ -0,0 +1,56 @@ +namespace Bts.DataStream; + +using BCG; + +using Bts.DataStreamCore; + +using mbts.utils; + +using System.Collections; +using System.Collections.Generic; +using System.Linq; + +public class BlueIndexedColumnTable : IBlueRowCollection { + readonly IBlueTableSchema schema; + readonly IList[] columnValues; + readonly Dictionary rowIdToIndex; + + public BlueIndexedColumnTable(IBlueTableSchema schema, IList[] columnValues) { + this.schema = schema; + this.columnValues = columnValues; + int idColumnIndex = schema.ResolveIndices("id"); + rowIdToIndex = columnValues[idColumnIndex] + .Select((idObj, idx) => new KeyValuePair(ExceptionAssert.Argument.Is(idObj), idx)) + .ToDictionary(); + } + + public IBlueRow this[int id] => + rowIdToIndex.TryGetValue(id, out var idx) + ? GetRow(idx) + : throw new KeyNotFoundException(); + + IBlueTableSchema IBlueRowCollection.schema => schema; + + public int Count => rowIdToIndex.Count; + + public bool containsID(int id) => + rowIdToIndex.ContainsKey(id); + + public IEnumerator GetEnumerator() { + int count = Count; + for (int i = 0; i < count; i++) { + yield return GetRow(i); + } + } + + public IBlueRow GetRow(int index) => + new BlueRowColumnWrapper(schema, index, columnValues); + + public int IndexOf(IBlueRow item) => + rowIdToIndex.TryGetValue(item.Id(), out var idx) && BlueRowEqualityComparer.Equals(GetRow(idx), item) + ? idx + : -1; + + IEnumerator IEnumerable.GetEnumerator() => + GetEnumerator(); +} diff --git a/src/DataStreamSuite/DataStream/BlueOrb/BlueRowColumnWrapper.cs b/src/DataStreamSuite/DataStream/BlueOrb/BlueRowColumnWrapper.cs new file mode 100644 index 000000000..b7798fade --- /dev/null +++ b/src/DataStreamSuite/DataStream/BlueOrb/BlueRowColumnWrapper.cs @@ -0,0 +1,21 @@ +namespace Bts.DataStream; + +using BCG; + +using System.Collections.Generic; + +public class BlueRowColumnWrapper : IBlueRow { + readonly int index; + readonly IList[] columns; + + public BlueRowColumnWrapper(IBlueTableSchema schema, int index, IList[] columns) { + this.schema = schema; + this.index = index; + this.columns = columns; + } + + public object? this[int index] => + columns[index][this.index]; + + public IBlueTableSchema schema { get; } +} diff --git a/src/DataStreamSuite/DataStream/BlueOrb/Parquet/IParquetColumnReader.cs b/src/DataStreamSuite/DataStream/BlueOrb/Parquet/IParquetColumnReader.cs new file mode 100644 index 000000000..62cab7a9c --- /dev/null +++ b/src/DataStreamSuite/DataStream/BlueOrb/Parquet/IParquetColumnReader.cs @@ -0,0 +1,29 @@ +namespace Bts.DataStream.Parquet; + +using bts.collections; + +using System; +using System.Collections.Generic; + +public interface IParquetColumnReader : IDisposable { + IList Read(); + IList Read(int? start, int? stop); +} + +public interface IParquetColumnReader : IParquetColumnReader { + new IList Read(); + new IList Read(int? start, int? stop); + + IList IParquetColumnReader.Read() => + Read().WrapSelector(v => (object?)v, null); + + IList IParquetColumnReader.Read(int? start, int? stop) => + Read(start, stop).WrapSelector(v => (object?)v, null); +} + +public interface IParquetColumnReader : IParquetColumnReader { + new IList Read(); + + IList IParquetColumnReader.Read() => + Read().WrapSelector(v => (object?)v, null); +} diff --git a/src/DataStreamSuite/DataStream/BlueOrb/Parquet/ParquetAdapter.cs b/src/DataStreamSuite/DataStream/BlueOrb/Parquet/ParquetAdapter.cs new file mode 100644 index 000000000..c4b8c5cda --- /dev/null +++ b/src/DataStreamSuite/DataStream/BlueOrb/Parquet/ParquetAdapter.cs @@ -0,0 +1,507 @@ +namespace Bts.DataStream.Parquet; + +using bts; +using bts.collections; + +using mbts.utils; + +using ParquetSharp; + +using Spark.Core.Collections; + +using System; +using System.Collections.Generic; + +public static class ParquetAdapter { + public static IParquetColumnReader GetColumnReader(this RowGroupReader rowGroupReader, int index, Type? colType = null) { + int rowCount = checked((int)rowGroupReader.MetaData.NumRows); + var colReader = rowGroupReader.Column(index); + return getColumnReader(colReader, rowCount, colType: colType); + } + + public static IList[] ReadColumns(this ParquetFileReader fileReader) { + var schema = fileReader.FileMetaData.Schema.ToBlueSchema("parquetReader"); + var colInfos = schema.columnInfo; + var colTypes = Array.ConvertAll(colInfos, c => c.colType); + var colValues = new IList[fileReader.FileMetaData.NumColumns]; + int rowCount = checked((int)fileReader.FileMetaData.NumRows); + for (int i = 0; i < colValues.Length; i++) { + colValues[i] = new List(rowCount); + } + int rowGroupCount = fileReader.FileMetaData.NumRowGroups; + for (int rowGroup = 0; rowGroup < rowGroupCount; ++rowGroup) { + using var rowGroupReader = fileReader.RowGroup(rowGroup); + rowGroupReader.ReadColumnsTo(ref colValues, colTypes); + } + return colValues; + } + + public static IList[] ReadColumns(this ParquetFileReader fileReader, int? start, int? stop) { + if (!start.HasValue && !stop.HasValue) { + return ReadColumns(fileReader); + } + var schema = fileReader.FileMetaData.ToBlueSchema("parquetReader"); + var colInfos = schema.columnInfo; + var colTypes = Array.ConvertAll(colInfos, c => c.colType); + var colValues = new IList[fileReader.FileMetaData.NumColumns]; + int rowCount = checked((int)fileReader.FileMetaData.NumRows); + start ??= 0; + stop ??= rowCount; + var sliceLength = stop.Value - start.Value; + for (int i = 0; i < colValues.Length; i++) { + colValues[i] = new List(sliceLength); + } + int numRowGroups = fileReader.FileMetaData.NumRowGroups; + int? numRowsPerGroup = null; + int rowIdx = 0; + for (int rowGroup = 0; rowGroup < numRowGroups; ++rowGroup) { + if (stop.Value <= rowIdx) { + break; + } + using var rowGroupReader = fileReader.RowGroup(rowGroup); + int numRowsInGroup = checked((int)rowGroupReader.MetaData.NumRows); + numRowsPerGroup ??= numRowsInGroup; + if (rowGroup < numRowGroups - 1) { + Log.assert(numRowsInGroup == numRowsPerGroup.Value); + } else { + Log.assert(numRowsInGroup <= numRowsPerGroup.Value); + } + int startIdx = start.Value - rowIdx; + int stopIdx = stop.Value - rowIdx; + rowIdx += numRowsInGroup; + if (startIdx >= numRowsInGroup) { + continue; + } + startIdx = Math.Max(0, startIdx); + stopIdx = Math.Min(stopIdx, numRowsInGroup); + int n = rowGroupReader.ReadColumnsTo(ref colValues, colTypes, startIdx, stopIdx); + Log.assert(n == (stopIdx - startIdx)); + } + return colValues; + } + + public static int ReadColumnsTo(this RowGroupReader rowGroupReader, ref IList[] columnValues, Type[] colTypes) { + int rowCount = checked((int)rowGroupReader.MetaData.NumRows); + for (int colIdx = rowGroupReader.MetaData.NumColumns - 1; colIdx > -1; colIdx--) { + using var colReader = rowGroupReader.GetColumnReader(colIdx, colTypes[colIdx]); + columnValues[colIdx].AddRange(colReader.Read()); + } + return rowCount; + } + + public static int ReadColumnsTo(this RowGroupReader rowGroupReader, ref IList[] columnValues, Type[] colTypes, int? start, int? stop) { + if (!start.HasValue && !stop.HasValue) { + return ReadColumnsTo(rowGroupReader, ref columnValues, colTypes); + } + int numRows = checked((int)rowGroupReader.MetaData.NumRows); + if (start.HasValue && start.Value >= numRows) { + return 0; + } + int? readRowCount = null; + for (int colIdx = rowGroupReader.MetaData.NumColumns - 1; colIdx > -1; colIdx--) { + using var colReader = rowGroupReader.GetColumnReader(colIdx, colTypes[colIdx]); + var colVals = colReader.Read(start, stop); + readRowCount ??= colVals.Count; + Log.assert(readRowCount == colVals.Count); + columnValues[colIdx].AddRange(colVals); + } + return readRowCount ?? throw new ArgumentException("Row group has no columns", nameof(rowGroupReader)); + } + + static IParquetColumnReader getColumnReader(ColumnReader reader, int bufferSize = 4096, Type? colType = null) { + var desc = reader.ColumnDescriptor; + var logicalType = desc.LogicalType; + return logicalType.Type switch { + LogicalTypeEnum.Int => IntReader(), + LogicalTypeEnum.String => new ParquetColumnReader(reader, bufferSize), + LogicalTypeEnum.Date => new ParquetColumnReader(reader, bufferSize), + LogicalTypeEnum.Decimal => DecimalReader(), + LogicalTypeEnum.Timestamp => TimestampReader(), + LogicalTypeEnum.None => DefaultReader(), + _ => throw new NotImplementedException(), + }; + + IParquetColumnReader DefaultReader() { + return desc.PhysicalType switch { + PhysicalType.Boolean => new ParquetColumnReader(reader, bufferSize), + PhysicalType.Int32 => new ParquetColumnReader(reader, bufferSize), + PhysicalType.Int64 => new ParquetColumnReader(reader, bufferSize), + PhysicalType.Float => new ParquetColumnReader(reader, bufferSize), + PhysicalType.Double => new ParquetColumnReader(reader, bufferSize), + _ => throw new NotImplementedException(), + }; + } + + IParquetColumnReader TimestampReader() { + var tlt = ExceptionAssert.of_type(logicalType); + return new ParquetColumnReader(reader, dtn => dtn?.DateTime, bufferSize); + } + + IParquetColumnReader IntReader() { + var ilt = ExceptionAssert.of_type(logicalType); + if (!ilt.IsSigned) { + if (colType is null) { + return ilt.BitWidth switch { + 8 => new ParquetColumnReader(reader, bufferSize), + 16 => new ParquetColumnReader(reader, bufferSize), + 32 => new ParquetColumnReader(reader, bufferSize), + 64 => new ParquetColumnReader(reader, bufferSize), + _ => throw new NotImplementedException(), + }; + } else if (colType == typeof(byte)) { + return ilt.BitWidth switch { + 8 => new ParquetColumnReader(reader, bufferSize), + 16 => new ParquetColumnReader(reader, v => v.HasValue ? checked((byte)v.Value) : null, bufferSize), + 32 => new ParquetColumnReader(reader, v => v.HasValue ? checked((byte)v.Value) : null, bufferSize), + 64 => new ParquetColumnReader(reader, v => v.HasValue ? checked((byte)v.Value) : null, bufferSize), + _ => throw new NotImplementedException(), + }; + } else if (colType == typeof(ushort)) { + return ilt.BitWidth switch { + 8 => new ParquetColumnReader(reader, v => v.HasValue ? v.Value : null, bufferSize), + 16 => new ParquetColumnReader(reader, bufferSize), + 32 => new ParquetColumnReader(reader, v => v.HasValue ? checked((ushort)v.Value) : null, bufferSize), + 64 => new ParquetColumnReader(reader, v => v.HasValue ? checked((ushort)v.Value) : null, bufferSize), + _ => throw new NotImplementedException(), + }; + } else if (colType == typeof(uint)) { + return ilt.BitWidth switch { + 8 => new ParquetColumnReader(reader, v => v.HasValue ? v.Value : null, bufferSize), + 16 => new ParquetColumnReader(reader, v => v.HasValue ? v.Value : null, bufferSize), + 32 => new ParquetColumnReader(reader, bufferSize), + 64 => new ParquetColumnReader(reader, v => v.HasValue ? checked((uint)v.Value) : null, bufferSize), + _ => throw new NotImplementedException(), + }; + } else if (colType == typeof(ulong)) { + return ilt.BitWidth switch { + 8 => new ParquetColumnReader(reader, v => v.HasValue ? v.Value : null, bufferSize), + 16 => new ParquetColumnReader(reader, v => v.HasValue ? v.Value : null, bufferSize), + 32 => new ParquetColumnReader(reader, v => v.HasValue ? v.Value : null, bufferSize), + 64 => new ParquetColumnReader(reader, bufferSize), + _ => throw new NotImplementedException(), + }; + } + throw new NotImplementedException($"Expected signed int column type, but got {colType}"); + } else { + if (colType is null) { + return ilt.BitWidth switch { + 8 => new ParquetColumnReader(reader, bufferSize), + 16 => new ParquetColumnReader(reader, bufferSize), + 32 => new ParquetColumnReader(reader, bufferSize), + 64 => new ParquetColumnReader(reader, bufferSize), + _ => throw new NotImplementedException(), + }; + } else if (colType == typeof(sbyte)) { + return ilt.BitWidth switch { + 8 => new ParquetColumnReader(reader, bufferSize), + 16 => new ParquetColumnReader(reader, v => v.HasValue ? checked((sbyte)v.Value) : null, bufferSize), + 32 => new ParquetColumnReader(reader, v => v.HasValue ? checked((sbyte)v.Value) : null, bufferSize), + 64 => new ParquetColumnReader(reader, v => v.HasValue ? checked((sbyte)v.Value) : null, bufferSize), + _ => throw new NotImplementedException(), + }; + } else if (colType == typeof(short)) { + return ilt.BitWidth switch { + 8 => new ParquetColumnReader(reader, v => v.HasValue ? checked((short)v.Value) : null, bufferSize), + 16 => new ParquetColumnReader(reader, bufferSize), + 32 => new ParquetColumnReader(reader, v => v.HasValue ? checked((short)v.Value) : null, bufferSize), + 64 => new ParquetColumnReader(reader, v => v.HasValue ? checked((short)v.Value) : null, bufferSize), + _ => throw new NotImplementedException(), + }; + } else if (colType == typeof(int)) { + return ilt.BitWidth switch { + 8 => new ParquetColumnReader(reader, v => v.HasValue ? v.Value : null, bufferSize), + 16 => new ParquetColumnReader(reader, v => v.HasValue ? v.Value : null, bufferSize), + 32 => new ParquetColumnReader(reader, bufferSize), + 64 => new ParquetColumnReader(reader, v => v.HasValue ? checked((int)v.Value) : null, bufferSize), + _ => throw new NotImplementedException(), + }; + } else if (colType == typeof(long)) { + return ilt.BitWidth switch { + 8 => new ParquetColumnReader(reader, v => v.HasValue ? v.Value : null, bufferSize), + 16 => new ParquetColumnReader(reader, v => v.HasValue ? v.Value : null, bufferSize), + 32 => new ParquetColumnReader(reader, v => v.HasValue ? v.Value : null, bufferSize), + 64 => new ParquetColumnReader(reader, bufferSize), + _ => throw new NotImplementedException(), + }; + } + throw new NotImplementedException($"Expected unsigned int column type, but got {colType}"); + } + } + + IParquetColumnReader DecimalReader() { + var dlt = ExceptionAssert.of_type(logicalType); + return new ParquetColumnReader(reader, bufferSize); + } + } + + public static IColumnWriter GetColumnWriter(TypeCode typeCode, ColumnWriter writer, int bufferSize = 4096) { + return typeCode switch { + TypeCode.Boolean => new ColumnWriter(writer, bufferSize), + TypeCode.SByte => new ColumnWriter(writer, bufferSize, GetInt8Converter((IntLogicalType)writer.ColumnDescriptor.LogicalType)), + TypeCode.Int16 => new ColumnWriter(writer, bufferSize, GetInt16Converter((IntLogicalType)writer.ColumnDescriptor.LogicalType)), + TypeCode.Int32 => new ColumnWriter(writer, bufferSize, GetInt32Converter((IntLogicalType)writer.ColumnDescriptor.LogicalType)), + TypeCode.Int64 => new ColumnWriter(writer, bufferSize, GetInt64Converter((IntLogicalType)writer.ColumnDescriptor.LogicalType)), + TypeCode.Byte => new ColumnWriter(writer, bufferSize, GetUInt8Converter((IntLogicalType)writer.ColumnDescriptor.LogicalType)), + TypeCode.UInt16 => new ColumnWriter(writer, bufferSize, GetUInt16Converter((IntLogicalType)writer.ColumnDescriptor.LogicalType)), + TypeCode.UInt32 => new ColumnWriter(writer, bufferSize, GetUInt32Converter((IntLogicalType)writer.ColumnDescriptor.LogicalType)), + TypeCode.UInt64 => new ColumnWriter(writer, bufferSize, GetUInt64Converter((IntLogicalType)writer.ColumnDescriptor.LogicalType)), + TypeCode.Single => new ColumnWriter(writer, bufferSize), + TypeCode.Double => new ColumnWriter(writer, bufferSize), + TypeCode.DateTime => new ColumnWriter(writer, bufferSize, ConvertDateTimeObj), + TypeCode.String => new ColumnWriter(writer, bufferSize), + _ => throw new NotSupportedException($"Unsupported column TypeCode {typeCode}"), + }; + + static DateTimeNanos? ConvertDateTimeObj(object? obj) { + if (obj is DateTime dt) { + return new(dt); + } + if (obj is DateTimeNanos dtn) { + return dtn; + } + throw new InvalidCastException($"Unable to convert object of type {obj?.GetType()} to DateTimeNanos"); + } + } + + public static Func GetInt8Converter(IntLogicalType logicalType) => + GetInt8Converter(logicalType.IsSigned, logicalType.BitWidth); + + public static Func GetInt8Converter(bool isSigned, int bitWidth) => + isSigned + ? bitWidth switch { + 8 => obj => obj as sbyte?, + 16 => obj => obj is short v ? checked((sbyte)v) : null, + 32 => obj => obj is int v ? checked((sbyte)v) : null, + 64 => obj => obj is long v ? checked((sbyte)v) : null, + _ => throw new NotImplementedException(), + } + : bitWidth switch { + 8 => obj => obj is byte v ? checked((sbyte)v) : null, + 16 => obj => obj is ushort v ? checked((sbyte)v) : null, + 32 => obj => obj is uint v ? checked((sbyte)v) : null, + 64 => obj => obj is ulong v ? checked((sbyte)v) : null, + _ => throw new NotImplementedException(), + }; + + public static Func GetInt16Converter(IntLogicalType logicalType) => + GetInt16Converter(logicalType.IsSigned, logicalType.BitWidth); + + public static Func GetInt16Converter(bool isSigned, int bitWidth) => + isSigned + ? bitWidth switch { + 8 => obj => obj is sbyte v ? v : null, + 16 => obj => obj as short?, + 32 => obj => obj is int v ? checked((short)v) : null, + 64 => obj => obj is long v ? checked((short)v) : null, + _ => throw new NotImplementedException(), + } + : bitWidth switch { + 8 => obj => obj is byte v ? v : null, + 16 => obj => obj is ushort v ? checked((short)v) : null, + 32 => obj => obj is uint v ? checked((short)v) : null, + 64 => obj => obj is ulong v ? checked((short)v) : null, + _ => throw new NotImplementedException(), + }; + + public static Func GetInt32Converter(IntLogicalType logicalType) => + GetInt32Converter(logicalType.IsSigned, logicalType.BitWidth); + + public static Func GetInt32Converter(bool isSigned, int bitWidth) => + isSigned + ? bitWidth switch { + 8 => obj => obj is sbyte v ? v : null, + 16 => obj => obj is short v ? v : null, + 32 => obj => obj as int?, + 64 => obj => obj is long v ? checked((int)v) : null, + _ => throw new NotImplementedException(), + } + : bitWidth switch { + 8 => obj => obj is byte v ? v : null, + 16 => obj => obj is ushort v ? v : null, + 32 => obj => obj is uint v ? checked((int)v) : null, + 64 => obj => obj is ulong v ? checked((int)v) : null, + _ => throw new NotImplementedException(), + }; + + public static Func GetInt64Converter(IntLogicalType logicalType) => + GetInt64Converter(logicalType.IsSigned, logicalType.BitWidth); + + public static Func GetInt64Converter(bool isSigned, int bitWidth) => + isSigned + ? bitWidth switch { + 8 => obj => obj is sbyte v ? v : null, + 16 => obj => obj is short v ? v : null, + 32 => obj => obj is int v ? v : null, + 64 => obj => obj as long?, + _ => throw new NotImplementedException(), + } + : bitWidth switch { + 8 => obj => obj is byte v ? v : null, + 16 => obj => obj is ushort v ? v : null, + 32 => obj => obj is uint v ? v : null, + 64 => obj => obj is ulong v ? checked((long)v) : null, + _ => throw new NotImplementedException(), + }; + + public static Func GetUInt8Converter(IntLogicalType logicalType) => + GetUInt8Converter(logicalType.IsSigned, logicalType.BitWidth); + + public static Func GetUInt8Converter(bool isSigned, int bitWidth) => + isSigned + ? bitWidth switch { + 8 => obj => obj is sbyte v ? checked((byte)v) : null, + 16 => obj => obj is short v ? checked((byte)v) : null, + 32 => obj => obj is int v ? checked((byte)v) : null, + 64 => obj => obj is long v ? checked((byte)v) : null, + _ => throw new NotImplementedException(), + } + : bitWidth switch { + 8 => obj => obj as byte?, + 16 => obj => obj is ushort v ? checked((byte)v) : null, + 32 => obj => obj is uint v ? checked((byte)v) : null, + 64 => obj => obj is ulong v ? checked((byte)v) : null, + _ => throw new NotImplementedException(), + }; + + public static Func GetUInt16Converter(IntLogicalType logicalType) => + GetUInt16Converter(logicalType.IsSigned, logicalType.BitWidth); + + public static Func GetUInt16Converter(bool isSigned, int bitWidth) => + isSigned + ? bitWidth switch { + 8 => obj => obj is sbyte v ? checked((ushort)v) : null, + 16 => obj => obj is short v ? checked((ushort)v) : null, + 32 => obj => obj is int v ? checked((ushort)v) : null, + 64 => obj => obj is long v ? checked((ushort)v) : null, + _ => throw new NotImplementedException(), + } + : bitWidth switch { + 8 => obj => obj is byte v ? v : null, + 16 => obj => obj is ushort v ? v : null, + 32 => obj => obj is uint v ? checked((ushort)v) : null, + 64 => obj => obj is ulong v ? checked((ushort)v) : null, + _ => throw new NotImplementedException(), + }; + + public static Func GetUInt32Converter(IntLogicalType logicalType) => + GetUInt32Converter(logicalType.IsSigned, logicalType.BitWidth); + + public static Func GetUInt32Converter(bool isSigned, int bitWidth) => + isSigned + ? bitWidth switch { + 8 => obj => obj is sbyte v ? checked((uint)v) : null, + 16 => obj => obj is short v ? checked((uint)v) : null, + 32 => obj => obj is int v ? checked((uint)v) : null, + 64 => obj => obj is long v ? checked((uint)v) : null, + _ => throw new NotImplementedException(), + } + : bitWidth switch { + 8 => obj => obj is byte v ? v : null, + 16 => obj => obj is ushort v ? v : null, + 32 => obj => obj is uint v ? v : null, + 64 => obj => obj is ulong v ? checked((uint)v) : null, + _ => throw new NotImplementedException(), + }; + + public static Func GetUInt64Converter(IntLogicalType logicalType) => + GetUInt64Converter(logicalType.IsSigned, logicalType.BitWidth); + + public static Func GetUInt64Converter(bool isSigned, int bitWidth) => + isSigned + ? bitWidth switch { + 8 => obj => obj is sbyte v ? checked((ulong)v) : null, + 16 => obj => obj is short v ? checked((ulong)v) : null, + 32 => obj => obj is int v ? checked((ulong)v) : null, + 64 => obj => obj is long v ? checked((ulong)v) : null, + _ => throw new NotImplementedException(), + } + : bitWidth switch { + 8 => obj => obj is byte v ? v : null, + 16 => obj => obj is ushort v ? v : null, + 32 => obj => obj is uint v ? v : null, + 64 => obj => obj is ulong v ? v : null, + _ => throw new NotImplementedException(), + }; + + public static string ColumnEnumTypeKey(string colName) => + $"{colName}.EnumType"; + + public static KeyValuePair ColumnEnumTypeEntry(string colName, Type enumType) { + if (enumType.IsNullable()) { + enumType = enumType.GetGenericArguments()[0]; + } + if (!enumType.IsEnum) { + throw new ArgumentException($"Must be enum type (actually {enumType})", nameof(enumType)); + } + return new(ColumnEnumTypeKey(colName), enumType.ToString()); + } + + public static Type? GetColumnEnumType(IReadOnlyDictionary? keyValueMetadata, string columnName) { + if (keyValueMetadata is not null) { + var key = ColumnEnumTypeKey(columnName); + if (keyValueMetadata.TryGetValue(key, out var typeName)) { + return Type.GetType(typeName); + } + } + return null; + } + + public interface IColumnWriter : IDisposable { + void Write(IReadOnlyCollection values); + } + + public interface IColumnWriter : IColumnWriter { + void Write(IReadOnlyCollection values); + } + + class ColumnWriter : IColumnWriter { + readonly T?[] buffer; + readonly LogicalColumnWriter writer; + readonly Func convert; + + public ColumnWriter(ColumnWriter writer, int bufferSize) + : this(writer, new T?[bufferSize]) { } + + public ColumnWriter(ColumnWriter writer, int bufferSize, Func convert) + : this(writer, new T?[bufferSize], convert) { } + + public ColumnWriter(ColumnWriter writer, T?[] buffer) + : this(writer, buffer, obj => (T?)obj) { } + + public ColumnWriter(ColumnWriter writer, T?[] buffer, Func convert) { + this.buffer = buffer; + this.writer = writer.LogicalWriter(); + this.convert = convert; + } + + public void Write(IReadOnlyCollection values) { + if (buffer.Length == 0) { + throw new InvalidOperationException("Cannot write because buffer length is 0"); + } + var count = values.Count; + var nPages = count / buffer.Length; + using var en = values.GetEnumerator(); + for (int pg = 0; pg < nPages; pg++) { + for (int i = 0; i < buffer.Length; i++) { + Log.assert(en.MoveNext(), "Enumeration failed before reaching expected count"); + buffer[i] = en.Current; + } + writer.WriteBatch(buffer!); + } + var remainder = count - nPages * buffer.Length; + if (remainder != 0) { + for (int i = 0; i < remainder; i++) { + Log.assert(en.MoveNext(), "Enumeration failed before reaching expected count"); + buffer[i] = en.Current; + } + writer.WriteBatch(buffer!, 0, remainder); + } + } + + public void Dispose() => + writer.Dispose(); + + void IColumnWriter.Write(IReadOnlyCollection values) => + Write(ReadOnlySelectorCollection.Create(values, convert)); + } +} diff --git a/src/DataStreamSuite/DataStream/BlueOrb/Parquet/ParquetColumnReader.cs b/src/DataStreamSuite/DataStream/BlueOrb/Parquet/ParquetColumnReader.cs new file mode 100644 index 000000000..5f2350c5d --- /dev/null +++ b/src/DataStreamSuite/DataStream/BlueOrb/Parquet/ParquetColumnReader.cs @@ -0,0 +1,111 @@ +namespace Bts.DataStream.Parquet; + +using bts; +using bts.collections; + +using ParquetSharp; + +using System; +using System.Collections.Generic; + +class ParquetColumnReader : IParquetColumnReader { + protected readonly LogicalColumnReader reader; + protected readonly T?[] buffer; + + public ParquetColumnReader(ColumnReader reader, int bufferSize) + : this(reader, new T?[bufferSize]) { } + + protected ParquetColumnReader(ColumnReader reader, T?[] buffer) { + this.buffer = buffer; + this.reader = reader.LogicalReader(); + } + + ~ParquetColumnReader() => + Dispose(); + + public IList Read() { + if (buffer.Length == 0) { + throw new InvalidOperationException("Cannot write because buffer length is 0"); + } + int n = reader.ReadBatch(buffer!); + return buffer.WrapSubRange(0, n); + } + + public IList Read(int? start, int? stop) { + if (buffer.Length == 0) { + throw new InvalidOperationException("Cannot write because buffer length is 0"); + } + start ??= 0; + stop ??= buffer.Length; + int length = stop.Value - start.Value; + if (length <= 0) { + return new List(0); + } + if (start.Value > 0) { + var nSkipped = checked((int)reader.Skip(start.Value)); + Log.assert(nSkipped >= 0 && nSkipped <= start.Value); + if (nSkipped < start.Value) { + return new List(0); + } + } + int n = reader.ReadBatch(buffer!, 0, length); + return buffer.WrapSubRange(0, n); + } + + public void Dispose() { + GC.SuppressFinalize(this); + reader.Dispose(); + } +} + +class ParquetColumnReader : IParquetColumnReader { + readonly Func convert; + readonly LogicalColumnReader reader; + readonly TIn[] buffer; + + public ParquetColumnReader(ColumnReader reader, Func convert, int bufferSize) + : this(reader, convert, new TIn[bufferSize]) { } + + public ParquetColumnReader(ColumnReader reader, Func convert, TIn[] buffer) { + this.convert = convert; + this.reader = reader.LogicalReader(); + this.buffer = buffer; + } + + ~ParquetColumnReader() => + Dispose(); + + public IList Read() { + if (buffer.Length == 0) { + throw new InvalidOperationException("Cannot write because buffer length is 0"); + } + int n = reader.ReadBatch(buffer!); + return buffer.WrapSelector(convert, null).WrapSubRange(0, n); + } + + public IList Read(int? start, int? stop) { + if (buffer.Length == 0) { + throw new InvalidOperationException("Cannot write because buffer length is 0"); + } + start ??= 0; + stop ??= buffer.Length; + int length = stop.Value - start.Value; + if (length <= 0) { + return new List(0); + } + if (start.Value > 0) { + var nSkipped = checked((int)reader.Skip(start.Value)); + Log.assert(nSkipped >= 0 && nSkipped <= start.Value); + if (nSkipped < start.Value) { + return new List(0); + } + } + int n = reader.ReadBatch(buffer!, 0, length); + return buffer.WrapSelector(convert, null).WrapSubRange(0, n); + } + + public void Dispose() { + GC.SuppressFinalize(this); + reader.Dispose(); + } +} diff --git a/src/DataStreamSuite/DataStream/BlueOrb/Parquet/ParquetExtensions.cs b/src/DataStreamSuite/DataStream/BlueOrb/Parquet/ParquetExtensions.cs new file mode 100644 index 000000000..ddf5d72af --- /dev/null +++ b/src/DataStreamSuite/DataStream/BlueOrb/Parquet/ParquetExtensions.cs @@ -0,0 +1,159 @@ +namespace Bts.DataStream.Parquet; + +using BCG; + +using bts; +using bts.collections; + +using Microsoft.Scripting.Utils; + +using ParquetSharp; + +using System; +using System.Collections.Generic; +using System.Linq; + +public static class ParquetExtensions { + public static int RowCount(this FileMetaData fileMetaData) => + checked((int)fileMetaData.NumRows); + + public static IBlueTableSchema ToBlueSchema(this SchemaDescriptor schemaDescriptor, string tableName = "parquet", IReadOnlyDictionary? keyValueMetadata = null) { + var colInfos = new ColumnInfo[schemaDescriptor.NumColumns]; + for (int i = 0; i < colInfos.Length; i++) { + colInfos[i] = schemaDescriptor.BlueColumn(i, keyValueMetadata); + } + return new BlueTableSchema(tableName, colInfos); + } + + public static IBlueTableSchema ToBlueSchema(this FileMetaData fileMetaData, string tableName = "parquet") => + ToBlueSchema(fileMetaData.Schema, tableName, fileMetaData.KeyValueMetadata); + + public static ColumnInfo BlueColumn(this SchemaDescriptor schemaDescriptor, int index, IReadOnlyDictionary? keyValueMetadata = null) { + var c = schemaDescriptor.Column(index); + var lt = c.LogicalType; + if (lt.Type == LogicalTypeEnum.Enum) { + var enumType = ParquetAdapter.GetColumnEnumType(keyValueMetadata, c.Name); + if (enumType is not null) { + return new(c.Name, index, enumType); + } + } + return new(c.Name, index, getTypeCode(c.LogicalType, c.PhysicalType)); + } + + public static IBlueRowCollection ReadBlueTable(this ParquetFileReader fileReader, int? start = null, int? stop = null) { + var schema = fileReader.FileMetaData.ToBlueSchema("parquetReader"); + var colInfos = schema.columnInfo; + IBlueRowCollection? rv; + if (!schema.columnInfo.Any(c => c.colName == "id")) { + rv = BlueAddIndexColumnTable.Create(schema, fileReader.ReadColumns(start, stop)); + } else { + rv = new BlueIndexedColumnTable(schema, fileReader.ReadColumns(start, stop)); + } + Log.assert(!schema.columnInfo.Any(c => c is null)); + return rv; + } + + public static (Column[] Column, IReadOnlyDictionary Meta) ToParquetSchema(this IBlueTableSchema schema) { + var blueColumns = schema.columnInfo; + var parquetColumns = new Column[blueColumns.Length]; + var parquetMeta = new Dictionary(); + KeyValuePair[] parquetColMeta; + for (int i = 0; i < blueColumns.Length; i++) { + (parquetColumns[i], parquetColMeta) = ToParquetColumn(blueColumns[i]); + parquetMeta.AddRange(parquetColMeta); + } + return (parquetColumns, parquetMeta); + } + + public static (Column Column, KeyValuePair[] Meta) ToParquetColumn(this ColumnInfo columnInfo) { + var colType = columnInfo.colType; + bool isNullable = colType.IsNullable(); + if (isNullable) { + colType = colType.GetGenericArguments()[0]; + } + KeyValuePair[] meta; + TypeCode typeCode; + var colName = columnInfo.colName; + if (colType.IsEnum) { + typeCode = colType.GetEnumUnderlyingType().GetTypeCode(); + meta = new[] { ParquetAdapter.ColumnEnumTypeEntry(columnInfo.colName, colType) }; + } else { + typeCode = colType.GetTypeCode(); + meta = Array.Empty>(); + } + return (ToParquetColumn(colName, typeCode), meta); + + static Column ToParquetColumn(string colName, TypeCode typeCode) => + typeCode switch { + TypeCode.Boolean => new Column(colName), + TypeCode.Byte => new Column(colName), + TypeCode.Int16 => new Column(colName), + TypeCode.Int32 => new Column(colName), + TypeCode.Int64 => new Column(colName), + TypeCode.UInt16 => new Column(colName), + TypeCode.UInt32 => new Column(colName), + TypeCode.UInt64 => new Column(colName), + TypeCode.Single => new Column(colName), + TypeCode.Double => new Column(colName), + TypeCode.DateTime => new Column(colName), + TypeCode.String => new Column(colName), + _ => throw new NotSupportedException($"Unsupported column TypeCode {typeCode}"), + }; + } + + static TypeCode getTypeCode(LogicalType logicalType, PhysicalType physicalType) { + return logicalType.Type switch { + // LogicalTypeEnum.Int => TypeCode.Int32, + // LogicalTypeEnum.Decimal => TypeCode.Double, + LogicalTypeEnum.Date or LogicalTypeEnum.Time or LogicalTypeEnum.Timestamp => TypeCode.DateTime, + LogicalTypeEnum.String => TypeCode.String, + LogicalTypeEnum.Int => FromInt(), + LogicalTypeEnum.Decimal => FromDecimal(), + _ => FromPhysicalType(), + }; + + TypeCode FromPhysicalType() => + physicalType switch { + PhysicalType.Boolean => TypeCode.Boolean, + PhysicalType.Int32 => TypeCode.Int32, + PhysicalType.Int64 => TypeCode.Int64, + PhysicalType.Float => TypeCode.Single, + PhysicalType.Double => TypeCode.Double, + _ => throw new NotImplementedException($"TypeCode not implemented for {logicalType} (logical) {physicalType} (physical)"), + }; + + TypeCode FromInt() { + var ilt = (IntLogicalType)logicalType; + var bitWidth = ilt.BitWidth; + return ilt.IsSigned ? FromSignedInt() : FromUnsignedInt(); + + TypeCode FromSignedInt() => + bitWidth switch { + 8 => TypeCode.SByte, + 16 => TypeCode.Int16, + 32 => TypeCode.Int32, + 64 => TypeCode.Int64, + _ => throw new NotSupportedException($"Bit width not supported for signed int ({bitWidth})"), + }; + + TypeCode FromUnsignedInt() => + bitWidth switch { + 8 => TypeCode.Byte, + 16 => TypeCode.UInt16, + 32 => TypeCode.UInt32, + 64 => TypeCode.UInt64, + _ => throw new NotSupportedException($"Bit width not supported for unsigned int ({bitWidth})"), + }; + } + + TypeCode FromDecimal() { + var dlt = (DecimalLogicalType)logicalType; + var scale = dlt.Scale; + var precision = dlt.Precision; + return scale switch { + 0 => throw new NotSupportedException($"Scale not supported for decimal ({scale})"), + _ => TypeCode.Double, + }; + } + } +} diff --git a/src/DataStreamSuite/DataStream/BlueOrb/Parquet/ParquetGroupTable.cs b/src/DataStreamSuite/DataStream/BlueOrb/Parquet/ParquetGroupTable.cs new file mode 100644 index 000000000..08545a581 --- /dev/null +++ b/src/DataStreamSuite/DataStream/BlueOrb/Parquet/ParquetGroupTable.cs @@ -0,0 +1,234 @@ +namespace Bts.DataStream.Parquet; + +using BCG; + +using bts; +using bts.collections; + +using Bts.DataStreamCore; + +using ParquetSharp; + +using Spark.Core.Collections; + +using System; +using System.Collections; +using System.Collections.Generic; +using System.Diagnostics.CodeAnalysis; + +public class ParquetGroupTable : IBlueRowCollection, IDisposable { + readonly RowGroupReader reader; + readonly IBlueTableSchema schema; + readonly bool leaveOpen; + readonly int fullCount; + readonly object syncRoot = new(); + readonly int? start; + readonly int? stop; + readonly int startIndex; + readonly int stopIndex; + readonly int idColumnIndex; + readonly int idColumnIndexInFile; + readonly Func? rowIdFactory; + DataCache? dataCacheBacking; + + public ParquetGroupTable( + RowGroupReader reader, + IBlueTableSchema schema, + int? start = null, + int? stop = null, + int idColumnIndex = -1, + int idColumnIndexInFile = -1, + Func? rowIdFactory = null, + bool leaveOpen = false + ) { + this.reader = reader; + this.schema = schema; + this.start = start; + this.stop = stop; + this.idColumnIndex = idColumnIndex; + this.idColumnIndexInFile = idColumnIndexInFile; + this.rowIdFactory = rowIdFactory; + this.leaveOpen = leaveOpen; + fullCount = checked((int)reader.MetaData.NumRows); + Log.assert(fullCount >= 0); + startIndex = SliceList.GetStartIndex(start, fullCount, true); + stopIndex = SliceList.GetStopIndex(stop, fullCount, true); + stopIndex = Math.Max(stopIndex, startIndex); + Log.assert( + startIndex >= 0 + && startIndex <= fullCount + && stopIndex >= 0 + && stopIndex <= fullCount + && stopIndex >= startIndex); + } + + ~ParquetGroupTable() => + Dispose(); + + public int Count => stopIndex - startIndex; + + IBlueTableSchema IBlueRowCollection.schema => schema; + + DataCache dataCache => + EnsureInit(); + + public IBlueRow this[int id] => dataCache[id]; + + public bool containsID(int id) => + dataCache.RowIdToIndex.ContainsKey(id); + + public IBlueRow GetRow(int index) => + dataCache.Rows[index]; + + public int IndexOf(IBlueRow item) { + int id = item.Id(); + return dataCache.RowIdToIndex.TryGetValue(id, out var idx) && BlueRowEqualityComparer.Equals(dataCache.Rows[idx], item) + ? idx + : -1; + } + + public bool TryGetValue(int id, [NotNullWhen(true)] out IBlueRow? row) { + if (dataCache.RowIdToIndex.TryGetValue(id, out var idx)) { + row = dataCache.Rows[idx]; + return true; + } + row = null; + return false; + } + + public IEnumerator GetEnumerator() { + for (int i = 0; i < dataCache.Rows.Length; i++) { + yield return dataCache.Rows[i]; + } + } + + public void Dispose() { + GC.SuppressFinalize(this); + if (!leaveOpen) { + reader.Dispose(); + } + } + + [MemberNotNull(nameof(dataCacheBacking))] + internal DataCache EnsureInit() { + if (dataCacheBacking.HasValue) { + return dataCacheBacking.Value; + } + lock (syncRoot) { + if (dataCacheBacking.HasValue) { + return dataCacheBacking.Value; + } + var rows = new IBlueRow[Count]; + int numColumns = reader.MetaData.NumColumns; + var blueColumns = schema.columnInfo; + Log.assert((blueColumns.Length - numColumns) switch { + 0 or 1 => true, + _ => false, + }); + var columnValues = allocateColumnValues( + reader, blueColumns.Length, Count, idColumnIndex, idColumnIndexInFile, startIndex, stopIndex, rowIdFactory); + var rowIdToIndex = new Dictionary(); + for (int i = 0; i < rows.Length; i++) { + rows[i] = new BlueRowColumnWrapper(schema, i, columnValues); + } + dataCacheBacking = new(rows, rowIdToIndex, columnValues); + } + return dataCacheBacking.Value; + } + + static IList[] allocateColumnValues( + RowGroupReader reader, + int numColumns, + int numRows, + int idColIdx, + int idColIdxInFile, + int startIndex, + int stopIndex, + Func? idFactory + ) { + if (idColIdx < 0 || idColIdx >= numColumns) { + throw new ArgumentException("Id column index out of bounds", nameof(idColIdx)); + } + var columnValues = new IList[numColumns]; + if (idColIdxInFile == -1) { + // Generate ids (there is no id column in the file) + ArgumentNullException.ThrowIfNull(idFactory); + for (int i = 0; i < numColumns; i++) { + if (i < idColIdx) { + ReadColumn(i, i); + } else if (i == idColIdx) { + columnValues[idColIdx] = new StepList(0, 1, numRows).WrapSelector(idx => (object?)idFactory(idx), null); + } else { + ReadColumn(i - 1, i); + } + } + } else if (idFactory is not null) { + // Generate ids (skip id column in the file) + for (int i = 0; i < numColumns; i++) { + if (i == idColIdx) { + columnValues[idColIdx] = new StepList(0, 1, numRows).WrapSelector(idx => (object?)idFactory(idx), null); + } else { + if (i > idColIdx && i < idColIdxInFile) { + ReadColumn(i - 1, i); + } else if (i > idColIdxInFile && i < idColIdx) { + ReadColumn(i + 1, i); + } else { + ReadColumn(i, i); + } + } + } + } else { + // Get ids from the file (output table id column index may differ from file) + // Case 1) File id col index > output + // src cols: a, b, c, d, (id), e, f, ... + // dst cols: a, b, (id), c, d, e, f, ... + // + // Case 2) File id col index == output + // src cols: a, b, (id), c, d, e, f, ... + // dst cols: a, b, (id), c, d, e, f, ... + // + // Case 3) File id col index < output + // src cols: a, b, (id), c, d, e, f, ... + // dst cols: a, b, c, d, (id), e, f, ... + + for (int i = 0; i < numColumns; i++) { + if (i == idColIdx) { + ReadColumn(idColIdxInFile, idColIdx); + } else if (i > idColIdx && i < idColIdxInFile) { + ReadColumn(i - 1, i); + } else if (i > idColIdxInFile && i < idColIdx) { + ReadColumn(i, i - 1); + } else { + ReadColumn(i, i); + } + } + } + return columnValues; + + void ReadColumn(int srcIdx, int dstIdx) { + using var colReader = reader.GetColumnReader(srcIdx); + columnValues[dstIdx] = colReader.Read(startIndex, stopIndex); + } + } + + IEnumerator IEnumerable.GetEnumerator() => + GetEnumerator(); + + internal readonly struct DataCache { + public readonly IBlueRow[] Rows; + public readonly Dictionary RowIdToIndex; + public readonly IList[] ColumnValues; + + public DataCache(IBlueRow[] rows, Dictionary rowIdToIndex, IList[] columnValues) { + Rows = rows; + RowIdToIndex = rowIdToIndex; + ColumnValues = columnValues; + } + + public IBlueRow this[int id] => + Rows[RowIdToIndex[id]]; + + public IBlueRow GetRow(int index) => + Rows[index]; + } +} diff --git a/src/DataStreamSuite/DataStream/BlueOrb/Parquet/ParquetReaderStream.cs b/src/DataStreamSuite/DataStream/BlueOrb/Parquet/ParquetReaderStream.cs new file mode 100644 index 000000000..c2c8135e7 --- /dev/null +++ b/src/DataStreamSuite/DataStream/BlueOrb/Parquet/ParquetReaderStream.cs @@ -0,0 +1,65 @@ +namespace Bts.DataStream.Parquet; + +using BCG; + +using bts; +using bts.collections; + +using Bts.DataStreamCore; +using Bts.PubSub; + +using Spark.Core.Disposables; + +using System; +using System.Threading; + +[DataStreamUserRespresentation(false)] +public class ParquetReaderStream : ViewStream { + readonly string file; + readonly int? start; + readonly int? stop; + readonly IWorkScheduler workScheduler; + IBlueRowCollection? table; + IDisposable? workerSub; + + public ParquetReaderStream(string file, IWorkScheduler workScheduler) { + this.file = file; + this.workScheduler = workScheduler; + } + + public ParquetReaderStream(string file, int? start, int? stop, IWorkScheduler workScheduler) { + this.file = file; + this.start = start; + this.stop = stop; + this.workScheduler = workScheduler; + } + + protected override void myOnFirstSubscriber() { + Start(); + var worker = new AnonymousWorker(DoWork); + workScheduler.Add(worker); + workerSub = Disposable.Create(() => workScheduler.Remove(worker)); + + async void Start() { + try { + Volatile.Write(ref table, await ParquetTableReader.ReadAsync(file, start, stop)); + } catch (Exception ex) { + Log.DataFlow.error($"[parquet_reader_stream_err.json]{{ \"err\": \"{ex}\" }}"); + } + } + + void DoWork() { + var table = Volatile.Read(ref this.table); + if (table is null) { + return; + } + Disposable.dispose_thread_safe(ref workerSub); + if (table.schema.columnInfo.Length > 1) { + myFireEvent(new DataPacket(table, table, new BlueTableImmutable(table.schema))); + } + } + } + + protected override void myOnLastSubscriber() => + Disposable.dispose_thread_safe(ref workerSub); +} diff --git a/src/DataStreamSuite/DataStream/BlueOrb/Parquet/ParquetRow.cs b/src/DataStreamSuite/DataStream/BlueOrb/Parquet/ParquetRow.cs new file mode 100644 index 000000000..2745e3094 --- /dev/null +++ b/src/DataStreamSuite/DataStream/BlueOrb/Parquet/ParquetRow.cs @@ -0,0 +1,26 @@ +// namespace Bts.DataStream.Parquet; + +// using BCG; + +// using global::Parquet.Data; + +// public class ParquetRow : IBlueRow { +// readonly int id; +// readonly int rowIndex; +// readonly DataColumn[] data; + +// public ParquetRow(int id, int rowIndex, DataColumn[] data, IBlueTableSchema schema) { +// this.id = id; +// this.rowIndex = rowIndex; +// this.data = data; +// Schema = schema; +// } + +// public object? this[int index] => +// index == 0 +// ? id +// : data[index - 1].Data.GetValue(rowIndex); + +// public IBlueTableSchema Schema { get; } +// IBlueTableSchema IBlueRow.schema => Schema; +// } diff --git a/src/DataStreamSuite/DataStream/BlueOrb/Parquet/ParquetTable.cs b/src/DataStreamSuite/DataStream/BlueOrb/Parquet/ParquetTable.cs new file mode 100644 index 000000000..bb9ac66e2 --- /dev/null +++ b/src/DataStreamSuite/DataStream/BlueOrb/Parquet/ParquetTable.cs @@ -0,0 +1,145 @@ +namespace Bts.DataStream.Parquet; + +using BCG; + +using bts; +using bts.collections; + +using Bts.DataStreamCore; + +using ParquetSharp; + +using Spark.Core.Collections; + +using System; +using System.Collections; +using System.Collections.Generic; +using System.Linq; +using System.Threading.Tasks; + +public class ParquetTable : IBlueRowCollection, IDisposable { + readonly SortedList groups = new(); + readonly int fullCount; + readonly IBlueTableSchema schema; + readonly ParquetFileReader? reader; + readonly int startIndex; + readonly int stopIndex; + + public ParquetTable(ParquetFileReader reader, string tableName = "parquet", int? start = null, int? stop = null, bool leaveOpen = false) { + schema = reader.FileMetaData.ToBlueSchema(tableName); + int idColIdx; + if (schema.TryResolveIndices("id", out int idColIdxInFile)) { + idColIdx = idColIdxInFile; + } else { + var origCols = schema.columnInfo; + var cols = new ColumnInfo[origCols.Length + 1]; + idColIdx = 0; + cols[0] = new("id", 0, typeof(int)); + for (int i = 0; i < origCols.Length; i++) { + var c = origCols[i]; + cols[i + 1] = new(c.colName, i + 1, c.colType); + } + schema = new BlueTableSchema(cols, schema.tableName); + } + fullCount = checked((int)reader.FileMetaData.NumRows); + Log.assert(fullCount >= 0); + startIndex = SliceList.GetStartIndex(start, fullCount, true); + stopIndex = SliceList.GetStopIndex(stop, fullCount, true); + stopIndex = Math.Max(stopIndex, startIndex); + Log.assert(startIndex >= 0 && startIndex <= fullCount && stopIndex >= 0 && stopIndex <= fullCount && stopIndex >= startIndex); + int numRowGroups = reader.FileMetaData.NumRowGroups; + int rowGroupLastIdx = 0; + object syncRoot = new(); + for (int i = 0; i < numRowGroups && rowGroupLastIdx < Count; i++) { + int? grpStart = start.HasValue ? startIndex - rowGroupLastIdx : null; + int? grpStop = stop.HasValue ? stopIndex - rowGroupLastIdx : null; + int offset = rowGroupLastIdx; + var grp = new ParquetGroupTable(reader.RowGroup(i), schema, grpStart, grpStop, idColIdx, idColIdxInFile, GetAutoId); + if (grp.Count == 0) { + continue; + } + rowGroupLastIdx += grp.Count; + groups.Add(rowGroupLastIdx, grp); + + int GetAutoId(int index) => + index + offset; + } + Parallel.For(0, groups.Count, i => groups.GetValueAtIndex(i).EnsureInit()); + if (!leaveOpen) { + this.reader = reader; + } + } + + ~ParquetTable() => + Dispose(); + + public IBlueTableSchema Schema => schema; + + public int Count => stopIndex - startIndex; + + IBlueTableSchema IBlueRowCollection.schema => schema; + + public IBlueRow this[int index] { + get { + foreach (var grp in groups.Values) { + if (grp.TryGetValue(index, out var r)) { + return r; + } + } + throw new KeyNotFoundException(); + } + } + + public static ParquetTable FromFile(string path, int? start = null, int? stop = null, string tableName = "parquet") => + new(new(path), tableName, start, stop, false); + + public IBlueRow GetRow(int index) { + if (index < 0 || index >= Count) { + throw new IndexOutOfRangeException(); + } + int rowGroupIdx = groups.Keys.BinarySearch(index); + if (rowGroupIdx < 0) { + rowGroupIdx = ~rowGroupIdx; + } + Log.assert(rowGroupIdx <= groups.Count); + var grp = groups[rowGroupIdx]; + int idxWithinGrp = index; + if (rowGroupIdx > 0) { + idxWithinGrp -= groups.Keys[rowGroupIdx - 1]; + } + return grp[idxWithinGrp]; + } + + public bool containsID(int id) => + groups.Values.Any(rgl => rgl.containsID(id)); + + public int IndexOf(IBlueRow item) { + int idx; + int grpFirstIdx = 0; + foreach (var (grpLastIdx, grp) in groups) { + idx = grp.IndexOf(item); + if (idx >= 0) { + return idx + grpFirstIdx; + } + grpFirstIdx = grpLastIdx; + } + return -1; + } + + public IEnumerator GetEnumerator() { + foreach (var rgl in groups.Values) { + foreach (var r in rgl) { + yield return r; + } + } + } + + public void Dispose() { + GC.SuppressFinalize(this); + groups.Values.DisposeAll(); + reader?.Dispose(); + } + + IEnumerator IEnumerable.GetEnumerator() => + GetEnumerator(); +} diff --git a/src/DataStreamSuite/DataStream/BlueOrb/Parquet/ParquetTableReader.cs b/src/DataStreamSuite/DataStream/BlueOrb/Parquet/ParquetTableReader.cs new file mode 100644 index 000000000..f8d779f7d --- /dev/null +++ b/src/DataStreamSuite/DataStream/BlueOrb/Parquet/ParquetTableReader.cs @@ -0,0 +1,32 @@ +#nullable enable +namespace Bts.DataStream.Parquet; + +using BCG; + +using Bts.DataStreamCore; + +using ParquetSharp; + +using System.IO; +using System.Threading.Tasks; + +public class ParquetTableReader { + public static Task ReadAsync(Stream stream, bool leaveOpen = false, int? start = null, int? stop = null) => + Task.Run(() => Read(stream, leaveOpen, start, stop)); + + public static Task ReadAsync(string path, int? start = null, int? stop = null) => + Task.Run(() => Read(path, start, stop)); + + public static IBlueRowCollection Read(Stream stream, bool leaveOpen = false, int? start = null, int? stop = null) { + using var fileReader = new ParquetFileReader(stream, leaveOpen); + var rv = fileReader.ReadBlueTable(start, stop); + rv.schema.Validate(); + return rv; + } + + public static IBlueRowCollection Read(string path, int? start = null, int? stop = null) { + IBlueRowCollection rv = ParquetTable.FromFile(path, start, stop); + rv.schema.Validate(); + return rv; + } +} diff --git a/src/DataStreamSuite/DataStream/BlueOrb/Parquet/ParquetTableWriter.cs b/src/DataStreamSuite/DataStream/BlueOrb/Parquet/ParquetTableWriter.cs new file mode 100644 index 000000000..5da2bc494 --- /dev/null +++ b/src/DataStreamSuite/DataStream/BlueOrb/Parquet/ParquetTableWriter.cs @@ -0,0 +1,38 @@ +namespace Bts.DataStream.Parquet; + +using BCG; + +using Bts.DataStreamCore; + +using bts; + +using ParquetSharp; + +using System.IO; +using System.Threading.Tasks; + +public static class ParquetTableWriter { + public static Task WriteAsync(IBlueRowCollection blueRows, Stream outStream, bool leaveOpen = false) => + Task.Run(() => Write(blueRows, outStream, leaveOpen)); + + public static void Write(IBlueRowCollection blueRows, Stream outStream, bool leaveOpen = false) { + var (parquetColumns, parquetMeta) = blueRows.schema.ToParquetSchema(); + using var schema = Column.CreateSchemaNode(parquetColumns); + using var writerProps = createWriterProperties(); + using var writer = new ParquetFileWriter(outStream, schema, writerProps, keyValueMetadata: parquetMeta, leaveOpen: leaveOpen); + using var rowGroup = writer.AppendRowGroup(); + for (int i = 0; i < blueRows.schema.columnInfo.Length; i++) { + var col = blueRows.schema.columnInfo[i]; + Log.assert(col.colNo == i); + using var colWriter = ParquetAdapter.GetColumnWriter(col.type_code(), rowGroup.NextColumn()); + colWriter.Write(blueRows.GetColumn(i)); + } + writer.Close(); + } + + static WriterProperties createWriterProperties(Compression compression = Compression.Snappy) { + using var builder = new WriterPropertiesBuilder(); + builder.Compression(compression); + return builder.Build(); + } +} diff --git a/src/DataStreamSuite/DataStream/BlueOrb/Parquet/ParquetWriterStream.cs b/src/DataStreamSuite/DataStream/BlueOrb/Parquet/ParquetWriterStream.cs new file mode 100644 index 000000000..85354afda --- /dev/null +++ b/src/DataStreamSuite/DataStream/BlueOrb/Parquet/ParquetWriterStream.cs @@ -0,0 +1,87 @@ +namespace Bts.DataStream.Parquet; + +using BCG; + +using bts.collections; + +using Bts.DataStreamCore; +using Bts.PubSub; + +using mbts.utils; + +using Spark.Core.Disposables; +using Spark.Core.Threading; + +using System; +using System.IO; + +[DataStreamUserRespresentation(false)] +public class ParquetWriterStream : ViewStream { + readonly string file; + readonly ViewStream inputStream; + readonly IWorkScheduler workScheduler; + readonly AtomicGate gate = new(); + IDataPacket? pendingOutput; + IDisposable? dispose; + + public ParquetWriterStream(ViewStream inputStream, string file, IWorkScheduler workScheduler) { + this.inputStream = inputStream; + this.file = file; + this.workScheduler = workScheduler; + } + + protected override void myOnFirstSubscriber() { + BlueTableImmutable.DataPacketBuilder? outputBuilder = null; + bool isFirstUpdate = false; + var worker = new AnonymousWorker(DoWork); + inputStream.onChange += OnInputStreamChange; + workScheduler.Add(worker); + dispose = Disposable.Create(Dispose); + + void Dispose() { + workScheduler.Remove(worker); + inputStream.onChange -= OnInputStreamChange; + } + + void OnInputStreamChange(object? dpObj) { + var dp = ExceptionAssert.Argument.Is(dpObj); + using (gate.BeginTransaction()) { + outputBuilder ??= new BlueTableImmutable(dp.Schema).ToDataPacketBuilder(); + foreach (IBlueRow r in dp.addedRows) { + outputBuilder.AddOrUpdate(r); + } + foreach (IBlueRow r in dp.deletedRows) { + outputBuilder.Remove(r.Id()); + } + } + } + + async void DoWork() { + using (gate.BeginTransaction()) { + if (outputBuilder is null) { + return; + } + if (pendingOutput is not null) { + myFireEvent(pendingOutput); + pendingOutput = null; + } + var dp = outputBuilder.ToDataPacket(); + if (isFirstUpdate) { + isFirstUpdate = false; + } else if (!dp.HasChanges()) { + return; + } + { + using var outStream = File.OpenWrite(file); + await ParquetTableWriter.WriteAsync(dp.newRows, outStream); + } + // Fire the update at the beginning of the next doWork call so it happens on the + // 'work' thread. + pendingOutput = dp; + } + } + } + + protected override void myOnLastSubscriber() => + Disposable.dispose_thread_safe(ref dispose); +} diff --git a/src/DataStreamSuite/DataStream/DataStream.csproj b/src/DataStreamSuite/DataStream/DataStream.csproj index 6b2201d40..bebbe1511 100644 --- a/src/DataStreamSuite/DataStream/DataStream.csproj +++ b/src/DataStreamSuite/DataStream/DataStream.csproj @@ -22,7 +22,7 @@ - + diff --git a/src/DataStreamSuite/DataStream/DataStreamExtensions.cs b/src/DataStreamSuite/DataStream/DataStreamExtensions.cs index 9b6bdfab2..c6315ac4a 100644 --- a/src/DataStreamSuite/DataStream/DataStreamExtensions.cs +++ b/src/DataStreamSuite/DataStream/DataStreamExtensions.cs @@ -2,6 +2,7 @@ namespace Bts.DataStream; using BCG; +using bts; using bts.api.core; using Bts.DataStreamCore; @@ -15,6 +16,8 @@ using Spark.Core.Threading; using System; using System.Linq; using System.Reactive.Linq; +using System.Threading; +using System.Threading.Tasks; public static class DataStreamExtensions { public static ViewStream AsViewStream(this IObservable source) => @@ -214,5 +217,118 @@ public static class DataStreamExtensions { } }); + public static IObservable NonIncrementalAsyncTransform(this IObservable source, Func> transformAsync, IWorkScheduler workScheduler) { + const int STATE_IDLE = 0; + const int STATE_BUSY = 1; + const int STATE_DISPOSED = 2; + return Observable.Create(Subscribe); + IDisposable Subscribe(IObserver o) { + var syncRoot = new object(); + IBlueRowCollection? lastTableIn = null; + BlueTableImmutable? lastTableOut = null; + int nConcurrentDoWorks = 0; + int state = STATE_IDLE; + IDisposable? dispose = null; + dispose = Disposable.Create(source.Subscribe(OnNext), workScheduler.Subscribe(DoWork)); + return Disposable.Create(Dispose); + void Dispose() { + lock (syncRoot) { + switch (state) { + case STATE_DISPOSED: + return; + case STATE_BUSY: + state = STATE_DISPOSED; + return; + } + state = STATE_DISPOSED; + } + Disposable.dispose(ref dispose); + } + + void OnNext(IDataPacket value) => + Volatile.Write(ref lastTableIn, value.newRows); + + async void DoWork() { + lock (syncRoot) { + nConcurrentDoWorks++; + switch (state) { + case STATE_DISPOSED: + case STATE_BUSY: + return; + case STATE_IDLE: + break; + default: + Log.assert(false, $"Unexpected state ({StateName(state)}) when DoWork called"); + return; + } + state = STATE_BUSY; + } + var dp = await PublishTable(); + bool isDisposed = false; + lock (syncRoot) { + switch (state) { + case STATE_DISPOSED: + isDisposed = true; + break; + case STATE_BUSY: + break; + default: + Log.assert(false, $"Unexpected state ({StateName(state)}) after building output table (async)"); + return; + } + } + if (isDisposed) { + Disposable.dispose(ref dispose); + return; + } + if (dp is not null) { + o.OnNext(dp); + } + lock (syncRoot) { + nConcurrentDoWorks = 0; + switch (state) { + case STATE_DISPOSED: + isDisposed = true; + break; + case STATE_BUSY: + state = STATE_IDLE; + break; + default: + Log.assert(false, $"Unexpected state ({StateName(state)}) after firing output table"); + return; + } + } + if (isDisposed) { + Disposable.dispose(ref dispose); + } + + async Task PublishTable() { + var tableIn = Interlocked.Exchange(ref lastTableIn, null); + if (tableIn is null) { + return null; + } + var tableOut = await transformAsync(tableIn); + if (tableOut is null) { + return null; + } + lastTableOut ??= new(tableOut.schema); + var bd = lastTableOut.ToDataPacketBuilder(); + bd.Clear(); + foreach (IBlueRow r in tableOut) { + bd.AddOrUpdate(r, BlueRowEqualityComparer.Instance); + } + return bd.ToDataPacket(out lastTableOut); + } + } + + static string StateName(int state) => + state switch { + STATE_IDLE => "IDLE", + STATE_BUSY => "BUSY", + STATE_DISPOSED => "DISPOSED", + _ => $"STATE_{state}", + }; + } + } } diff --git a/src/DataStreamSuite/DataStream/Primitives/SideEffectStreams/CSVSaverStream.cs b/src/DataStreamSuite/DataStream/Primitives/SideEffectStreams/CSVSaverStream.cs index aa132b1b2..17ba329b5 100644 --- a/src/DataStreamSuite/DataStream/Primitives/SideEffectStreams/CSVSaverStream.cs +++ b/src/DataStreamSuite/DataStream/Primitives/SideEffectStreams/CSVSaverStream.cs @@ -5,8 +5,6 @@ using System.IO; using BCG.Utilities; using System.Diagnostics; using Microsoft.Data.Analysis; -using Parquet; -using Parquet.Data; namespace BCG { diff --git a/src/DataStreamSuite/DataStreamCore/BlueData/BlueColumnValueCollection.cs b/src/DataStreamSuite/DataStreamCore/BlueData/BlueColumnValueCollection.cs new file mode 100644 index 000000000..944ac1144 --- /dev/null +++ b/src/DataStreamSuite/DataStreamCore/BlueData/BlueColumnValueCollection.cs @@ -0,0 +1,34 @@ +#nullable enable +namespace Bts.DataStreamCore; + +using BCG; + +using Spark.Core.Collections; + +using System; +using System.Collections.Generic; + +public static class BlueColumnValueCollection { + public static BlueColumnValueCollection Create(IBlueRowCollection rows, int columnIndex, Func convert) => + new(rows, columnIndex, convert); +} + +public class BlueColumnValueCollection : IMutableCollectionFacade { + readonly IBlueRowCollection rows; + readonly int columnIndex; + readonly Func convert; + + public BlueColumnValueCollection(IBlueRowCollection rows, int columnIndex, Func convert) { + this.rows = rows; + this.columnIndex = columnIndex; + this.convert = convert; + } + + public int Count => rows.Count; + + public IEnumerator GetEnumerator() { + foreach (IBlueRow r in rows) { + yield return convert(r[columnIndex]); + } + } +} diff --git a/src/DataStreamSuite/DataStreamCore/BlueData/BlueRowCollectionExtensions.cs b/src/DataStreamSuite/DataStreamCore/BlueData/BlueRowCollectionExtensions.cs new file mode 100644 index 000000000..7e0af8134 --- /dev/null +++ b/src/DataStreamSuite/DataStreamCore/BlueData/BlueRowCollectionExtensions.cs @@ -0,0 +1,17 @@ +#nullable enable +namespace Bts.DataStreamCore; + +using BCG; + +using Spark.Core.Collections; + + +using System.Collections.Generic; + +public static class BlueRowCollectionExtensions { + public static IReadOnlyCollection GetColumn(this IBlueRowCollection rows, string columnName) => + rows.GetColumn(rows.schema.columnInfo.ResolveIndices(columnName)); + + public static IReadOnlyCollection GetColumn(this IBlueRowCollection rows, int columnIndex) => + SelectorCollection.Create(rows, r => ((IBlueRow)r!)[columnIndex]); +} diff --git a/src/DataStreamSuite/DataStreamCore/BlueData/BlueTableSchemaExtensions.cs b/src/DataStreamSuite/DataStreamCore/BlueData/BlueTableSchemaExtensions.cs index 745280109..4afe6dd71 100644 --- a/src/DataStreamSuite/DataStreamCore/BlueData/BlueTableSchemaExtensions.cs +++ b/src/DataStreamSuite/DataStreamCore/BlueData/BlueTableSchemaExtensions.cs @@ -2,9 +2,12 @@ namespace Bts.DataStreamCore; using BCG; +using bts.BlueOrb; using bts.collections; using System; +using System.Diagnostics.CodeAnalysis; +using System.Linq; public static class BlueTableSchemaExtensions { public static int ResolveIndices(this IBlueTableSchema schema, string name) { @@ -84,4 +87,33 @@ public static class BlueTableSchemaExtensions { } return indices; } + + public static IBlueTableSchema Validate([NotNullWhen(true)]this IBlueTableSchema? schema) { + if (schema is null) { + throw SchemaValidationExceptionFactory.Null(); + } + var cols = schema.columnInfo ?? throw SchemaValidationExceptionFactory.NullOrEmptyColumns(); + if (cols.Length == 0) { + throw SchemaValidationExceptionFactory.NoColumns(); + } + var colNames = schema.columnNames ?? throw SchemaValidationExceptionFactory.NullOrEmptyColumnNames(); + if (colNames.Any(string.IsNullOrWhiteSpace)) { + throw SchemaValidationExceptionFactory.NullEmptyWhitespaceColumnName(); + } + if (!schema.columnNames.SequenceEqual(schema.columnInfo.Select(c => c.colName))) { + throw SchemaValidationExceptionFactory.MismatchedColumnNames(); + } + if (!cols.Any(c => c.colName == "id" && c.colType == typeof(int))) { + throw SchemaValidationExceptionFactory.NoIdColumn(); + } + if (cols.Contains(null)) { + throw SchemaValidationExceptionFactory.NullColumn(); + } + for (int i = 0; i < cols.Length; i++) { + if (cols[i].colNo != i) { + throw SchemaValidationExceptionFactory.MismatchedColumnIndices(); + } + } + return schema; + } } diff --git a/src/DataStreamSuite/DataStreamCore/BlueData/SchemaValidationException.cs b/src/DataStreamSuite/DataStreamCore/BlueData/SchemaValidationException.cs new file mode 100644 index 000000000..a9d8b0eff --- /dev/null +++ b/src/DataStreamSuite/DataStreamCore/BlueData/SchemaValidationException.cs @@ -0,0 +1,42 @@ +namespace bts.BlueOrb; + +using System; + +public class SchemaValidationException : Exception { + public SchemaValidationException(string message) : base(message) { } +} + +static class SchemaValidationExceptionFactory { + public static SchemaValidationException NoColumns() => + new("IBlueTableSchema has no columns"); + + public static SchemaValidationException NoIdColumn() => + new("IBlueTableSchema has no id column"); + + public static SchemaValidationException IdColumnInvalidType(Type type) => + new($"IBlueTableSchema id column has invalid type {type}"); + + public static SchemaValidationException Null() => + new("IBlueTableSchema is null"); + + public static SchemaValidationException NullColumn() => + new("IBlueTableSchema has null column"); + + public static SchemaValidationException NullOrEmptyColumns() => + new("IBlueTableSchema columnInfo array is null/empty"); + + public static SchemaValidationException NullOrEmptyColumnNames() => + new("IBlueTableSchema columnNames array is null/empty"); + + public static SchemaValidationException NullEmptyWhitespaceColumnName() => + new("IBlueTableSchema columnName is null/empty/whitespace"); + + public static SchemaValidationException DuplicateColumnName() => + new("IBlueTableSchema has duplicate column names"); + + public static SchemaValidationException MismatchedColumnNames() => + new("IBlueTableSchema columnNames array does not match names in columnInfo array"); + + public static SchemaValidationException MismatchedColumnIndices() => + new("IBlueTableSchema column numbers do not match corresponding indices"); +} diff --git a/src/DataStreamSuite/SharedStreams/SharedStreams.csproj b/src/DataStreamSuite/SharedStreams/SharedStreams.csproj index 44f691af0..08a82f8b6 100644 --- a/src/DataStreamSuite/SharedStreams/SharedStreams.csproj +++ b/src/DataStreamSuite/SharedStreams/SharedStreams.csproj @@ -19,6 +19,7 @@ + diff --git a/src/client/spark_core/Collections/ArrayFactory.cs b/src/client/spark_core/Collections/ArrayFactory.cs index d877e55c8..090bef9d3 100644 --- a/src/client/spark_core/Collections/ArrayFactory.cs +++ b/src/client/spark_core/Collections/ArrayFactory.cs @@ -47,5 +47,13 @@ namespace bts.collections { } return array; } + + public static T[][] CreateJaggedTable(int m, int n) { + var table = new T[m][]; + while (m-- > 0) { + table[m] = new T[n]; + } + return table; + } } } diff --git a/src/client/spark_core/Collections/Extensions/ReadOnlyCollection.cs b/src/client/spark_core/Collections/Extensions/ReadOnlyCollection.cs new file mode 100644 index 000000000..1f014de0d --- /dev/null +++ b/src/client/spark_core/Collections/Extensions/ReadOnlyCollection.cs @@ -0,0 +1,42 @@ +#nullable enable +namespace Spark.Core.Collections; + +using System.Collections; +using System.Collections.Generic; + +public class ReadOnlyCollection : IReadOnlyCollection { + readonly ICollection source; + + public ReadOnlyCollection(ICollection source) => + this.source = source; + + public int Count => source.Count; + + public static ReadOnlyCollection Create(ICollection source) => + new(source); + + public IEnumerator GetEnumerator() { + foreach (var obj in source) { + yield return obj; + } + } + + IEnumerator IEnumerable.GetEnumerator() => + source.GetEnumerator(); +} + + +public class ReadOnlyCollection : IReadOnlyCollection { + readonly ICollection source; + + public ReadOnlyCollection(ICollection source) => + this.source = source; + + public int Count => source.Count; + + public IEnumerator GetEnumerator() => + source.GetEnumerator(); + + IEnumerator IEnumerable.GetEnumerator() => + source.GetEnumerator(); +} diff --git a/src/client/spark_core/Collections/Extensions/ReadOnlySelectorCollection.cs b/src/client/spark_core/Collections/Extensions/ReadOnlySelectorCollection.cs new file mode 100644 index 000000000..85997e451 --- /dev/null +++ b/src/client/spark_core/Collections/Extensions/ReadOnlySelectorCollection.cs @@ -0,0 +1,56 @@ +#nullable enable +namespace Spark.Core.Collections; + +using System; +using System.Collections; +using System.Collections.Generic; + +public static class ReadOnlySelectorCollection { + public static ReadOnlySelectorCollection Create(IReadOnlyCollection source, Func selector) => + new(source, selector); + + public static ReadOnlySelectorCollection Create(IReadOnlyCollection source, Func selector) => + new(source, selector); +} + +public class ReadOnlySelectorCollection : IReadOnlyCollection { + readonly IReadOnlyCollection source; + readonly Func selector; + + public ReadOnlySelectorCollection(IReadOnlyCollection source, Func selector) { + this.source = source; + this.selector = selector; + } + + public int Count => source.Count; + + public IEnumerator GetEnumerator() { + foreach (var obj in source) { + yield return selector(obj); + } + } + + IEnumerator IEnumerable.GetEnumerator() => + GetEnumerator(); +} + +public class ReadOnlySelectorCollection : IReadOnlyCollection { + readonly IReadOnlyCollection source; + readonly Func selector; + + public ReadOnlySelectorCollection(IReadOnlyCollection source, Func selector) { + this.source = source; + this.selector = selector; + } + + public int Count => source.Count; + + public IEnumerator GetEnumerator() { + foreach (var val in source) { + yield return selector(val); + } + } + + IEnumerator IEnumerable.GetEnumerator() => + GetEnumerator(); +} diff --git a/src/client/spark_core/Collections/Extensions/SelectorCollection.cs b/src/client/spark_core/Collections/Extensions/SelectorCollection.cs index 44b1b1971..64caea08d 100644 --- a/src/client/spark_core/Collections/Extensions/SelectorCollection.cs +++ b/src/client/spark_core/Collections/Extensions/SelectorCollection.cs @@ -7,12 +7,36 @@ using System.Collections.Generic; using System.Linq; public static class SelectorCollection { + public static SelectorCollection Create(ICollection source, Func convert) => + new(source, convert); + public static ICollection Create( ICollection source, Func convert, Func? convertBack = null) => new SelectorCollection(source, convert, convertBack); } -public class SelectorCollection : ICollection { +public class SelectorCollection : IMutableCollectionFacade { + readonly ICollection source; + readonly Func selector; + + public SelectorCollection(ICollection source, Func selector) { + this.source = source; + this.selector = selector; + } + + public int Count => source.Count; + + public IEnumerator GetEnumerator() { + foreach (var obj in source) { + yield return selector(obj); + } + } + + IEnumerator IEnumerable.GetEnumerator() => + GetEnumerator(); +} + +public class SelectorCollection : ICollection, IReadOnlyCollection { readonly ICollection source; readonly Func convert; readonly Func? convertBack; diff --git a/src/client/spark_core/Collections/IMutableCollectionFacade.cs b/src/client/spark_core/Collections/IMutableCollectionFacade.cs index ab05849c0..e66222a05 100644 --- a/src/client/spark_core/Collections/IMutableCollectionFacade.cs +++ b/src/client/spark_core/Collections/IMutableCollectionFacade.cs @@ -5,7 +5,7 @@ using System; using System.Collections; using System.Collections.Generic; -public interface IMutableCollectionFacade : ICollection { +public interface IMutableCollectionFacade : IReadOnlyCollection, ICollection { bool ICollection.IsReadOnly => true; void ICollection.Add(T item) => throw new NotSupportedException(); diff --git a/src/client/spark_core/utils/ArrayUtils.cs b/src/client/spark_core/utils/ArrayUtils.cs index c27bc88de..56ce38c70 100644 --- a/src/client/spark_core/utils/ArrayUtils.cs +++ b/src/client/spark_core/utils/ArrayUtils.cs @@ -1,4 +1,5 @@ using mbts.utils; + using System; using System.Collections; using System.Collections.Generic; @@ -115,6 +116,52 @@ namespace bts.utils { } } + public static void InsertExact(ref T[] array, int index, params T[] values) { + if (values.Length == 0) { + return; + } + if (index < 0 || index > array.Length) { + throw new IndexOutOfRangeException(); + } + if (array.Length == 0) { + array = values; + return; + } + var origArray = array; + array = new T[array.Length + values.Length]; + Array.Copy(origArray, 0, array, 0, index); + Array.Copy(values, 0, array, index, values.Length); + Array.Copy(origArray, index, array, index + values.Length, origArray.Length - index); + } + + public static T[] InsertExact(T[] array, int index, bool forceClone, params T[] values) { + if (index < 0 || index > array.Length) { + throw new IndexOutOfRangeException(); + } + T[] rv; + if (values.Length == 0) { + if (forceClone) { + rv = new T[array.Length]; + Array.Copy(array, rv, array.Length); + } else { + rv = array; + } + } else if (array.Length == 0) { + if (forceClone) { + rv = new T[values.Length]; + Array.Copy(values, rv, values.Length); + } else { + rv = values; + } + } else { + rv = new T[array.Length + values.Length]; + Array.Copy(array, 0, rv, 0, index); + Array.Copy(values, 0, rv, index, values.Length); + Array.Copy(array, index, rv, index + values.Length, array.Length - index); + } + return rv; + } + public static void remove_at(ref T[] array, int index) { int sz = array.Length; remove_at(ref array, ref sz, index);