Files
spark/src/DataStreamSuite/DataStream/Primitives/Project/New/NewProjectStream.cs
T

203 lines
7.4 KiB
C#

namespace bts {
using BCG;
using ProtoBuf;
using System;
using System.Collections;
using System.Collections.Generic;
using System.Linq;
public class ProjectData {
// schema from the base stream
public readonly IBlueTableSchema in_schema;
// this is the projected schema
public readonly IBlueTableSchema out_schema;
// projected cols to the input cols, for cols who's names are changed
public readonly IDictionary<string, string> mapped_to_orig_col_map = new Dictionary<string, string>();
public ProjectData(IBlueTableSchema in_schema_, IBlueTableSchema out_schema_,
IDictionary<string, string> mapped_to_orig_col_map_) {
in_schema = in_schema_;
out_schema = out_schema_;
mapped_to_orig_col_map = mapped_to_orig_col_map_;
}
}
public class NewProjectRow : IBlueRow {
#region IBlueRow members
private ProjectData _project_data;
private IBlueRow _row;
public NewProjectRow(ProjectData project_data, IBlueRow row) {
_project_data = project_data;
_row = row;
}
public IBlueTableSchema schema { get { return _project_data.out_schema; } }
#endregion
#region ICollection members
public int Count { get { return schema.columnNames.Length; } }
public bool IsSynchronized { get { return false; } }
public object SyncRoot { get { return this; } }
public void CopyTo(Array array, int index) {
int i = index;
foreach (object o in this as IEnumerable)
array.SetValue(o, i++);
}
#endregion
#region IDictionary members
public bool IsFixedSize { get { return true; } }
public bool IsReadOnly { get { return true; } }
public object this[object key] {
get {
if (key is string)
return this[(string)key];
return this[(int)key];
}
set { throw new NotSupportedException("The IBlueRow is readonly"); }
}
public object this[int index] {
get {
return this[schema.columnInfo[index].colName];
}
}
public object this[string col_name] {
get {
string mapped_to_col = null;
if (_project_data.mapped_to_orig_col_map.TryGetValue(col_name, out mapped_to_col)) {
return _row[mapped_to_col];
}
return _row[col_name];
}
}
public ICollection Keys { get { return schema.columnNames; } }
public ICollection Values {
get {
object[] retVal = new object[schema.columnNames.Length];
CopyTo(retVal, 0);
return retVal;
}
}
public void Add(object key, object value) { throw new NotSupportedException("The IBlueRow is readonly"); }
public void Clear() { throw new NotSupportedException("The IBlueRow is readonly"); }
public bool Contains(object key) { return schema[(string)key] != null; }
/// <summary>
/// Note, the default enumerator is the IDictionary's enumerator, so if user needs to
/// iterate through the values, typecast this into an IEnumerable
/// </summary>
/// <returns></returns>
public IDictionaryEnumerator GetEnumerator() { return new BlueRowDictionaryEnumerator(this); }
IEnumerator IEnumerable.GetEnumerator() { return new BlueRowEnumerator(this); }
public void Remove(object key) { throw new NotSupportedException("The IBlueRow is readonly"); }
#endregion
}
public class NewProjectStream : ViewStream, IStreamMapper {
private ViewStream _stream;
private string _project_expression;
private ProjectData _project_data = null;
//public ProjectData
public NewProjectStream(ViewStream vs, string project_expression) {
_stream = vs;
_project_expression = project_expression;
_stream = new MapStream(vs, this);
mySetRealSource(_stream);
}
public IBlueTableSchema getOutputSchema(IBlueTableSchema input_schema) {
_project_data = _create_project_data(input_schema, _project_expression);
return _project_data.out_schema;
}
public IBlueRow map(IBlueRow row) {
return new NewProjectRow(_project_data, row);
}
private static ProjectData _create_project_data(IBlueTableSchema input_schema,
string project_expression) {
project_expression = project_expression.Trim();
string[] project_toks = project_expression.Split(' ');
HashSet<string> removed_cols = new HashSet<string>();
Dictionary<string, string> mapped_to_orig_map = new Dictionary<string, string>();
Dictionary<string, string> orig_to_mapped = new Dictionary<string, string>();
// by default every col will show up, so if we have the projection as (a:a1)
// it will convert a to a1,
// examples: if a schema has cols a b c
// project(parent_str, "a:aa ") -> aa b c
// project(parent_str, "a:aa !c") -> aa b
// project(parent_str, "a:aa !") -> aa
// project(parent_str, "a:aa b = (aa*aa)!") -> aa b
bool hide_all = false;
foreach (string p_tok in project_toks) {
if (p_tok.StartsWith("!")) {
if (p_tok.Length == 1) {
// if it is a "blanket "!"=> hide all cols except the one that
// are stated explicitly
hide_all = true;
} else {
string removed_col = p_tok.Substring(1);
removed_cols.Add(removed_col);
}
} else if (p_tok.Contains(":")) {
string[] map_toks = p_tok.Split(":");
string orig_col = map_toks[0];
string mapped_col = map_toks[1];
mapped_to_orig_map.Add(mapped_col, orig_col);
orig_to_mapped.Add(orig_col, mapped_col);
} else {
if (input_schema.columnNames.Contains(p_tok)) {
string orig_col = p_tok;
string mapped_col = orig_col;
orig_to_mapped.Add(orig_col, mapped_col);
}
}
}
// make the new schema
List<ColumnInfo> new_cols = new List<ColumnInfo>();
foreach (ColumnInfo ci in input_schema.columnInfo) {
if (ci.colName == "id") {
ColumnInfo nc = new ColumnInfo(ci.colName, new_cols.Count, ci.colType);
new_cols.Add(nc);
continue;
}
if (removed_cols.Contains(ci.colName)) {
continue;
} else if (orig_to_mapped.ContainsKey(ci.colName)) {
string mapped_name = orig_to_mapped[ci.colName];
ColumnInfo nc = new ColumnInfo(mapped_name, new_cols.Count, ci.colType);
new_cols.Add(nc);
} else {
if (hide_all)
continue;
//if (ci.colName == "id" || removed_cols.Count > 0) {
// unless columns are explicitely hidden, all columns show up
ColumnInfo nc = new ColumnInfo(ci.colName, new_cols.Count, ci.colType);
new_cols.Add(nc);
//}
}
}
IBlueTableSchema new_schema = new BlueTableSchema(input_schema.tableName, new_cols.ToArray());
return new ProjectData(input_schema, new_schema, mapped_to_orig_map);
}
}
}