Files
spark/src/DataStreamSuite/DataStream/Primitives/MongoDB/MongoSchemaAdaptor.cs
T

56 lines
2.2 KiB
C#

namespace Bts.DataStream;
using bts;
using MongoDB.Bson;
using System;
using System.Collections.Generic;
public static class MongoSchemaAdaptor {
public static BlueTableSchemaAdaptor<BsonDocument> InferSchema(IEnumerable<BsonDocument> docs) {
var schemaAdaptor = new BlueTableSchemaAdaptor<BsonDocument>("MongoCollectionStream");
foreach (var doc in docs) {
foreach (var el in doc) {
if (schemaAdaptor.Contains(el.Name)) {
continue;
}
switch (el.Value.BsonType) {
case BsonType.Array:
schemaAdaptor.AddColumn(el.Name, d => d.GetArrayOrDefault(el.Name)?.ToString() ?? string.Empty);
break;
case BsonType.String:
schemaAdaptor.AddColumn(el.Name, d => d.GetStringOrDefault(el.Name) ?? string.Empty);
break;
case BsonType.Int32:
schemaAdaptor.AddColumn(el.Name, d => d.GetInt32OrDefault(el.Name));
break;
case BsonType.Int64:
schemaAdaptor.AddColumn(el.Name, d => d.GetInt64OrDefault(el.Name));
break;
case BsonType.Double:
schemaAdaptor.AddColumn(el.Name, d => d.GetDoubleOrDefault(el.Name));
break;
case BsonType.DateTime:
schemaAdaptor.AddColumn(el.Name, d => d.GetDateTimeOrDefault(el.Name));
break;
case BsonType.Timestamp:
schemaAdaptor.AddColumn(el.Name, d => d.GetTimestampOrDefault(el.Name));
break;
case BsonType.ObjectId:
schemaAdaptor.AddColumn(el.Name, d => d.GetObjectIdOrDefault(el.Name)?.ToString());
break;
case BsonType.Null:
Log.Gen.warn(
$"Ignoring column '{el.Name}' because at least one element is null");
break;
default:
throw new InvalidOperationException(
$"Unsupported BsonType '{el.Value.BsonType}'");
}
}
}
return schemaAdaptor;
}
}