Files
spark/src/DataStreamSuite/DataStreamCore/BlueData/FileRowCollection.cs
T
2023-09-04 16:25:40 -04:00

55 lines
1.7 KiB
C#

using System;
using System.IO;
using System.Collections;
namespace BCG
{
public class FileRowCollection : CircularRowCollection
{
public FileRowCollection(string dataFileName,
string schemaFileName)
: this(dataFileName, schemaFileName, Int32.MaxValue)
{ }
/// <summary>
/// </summary>
/// <param name="dataFileName">file with the rows</param>
/// <param name="schemFileName">file with schema (written as columns names in the first line and types in the 2nd</param>
/// <param name="maxSize">if max size is mentioned then a buffer is mantained for only
/// that number of rows, ie the last maxSize number of rows in the file
/// </param>
public FileRowCollection(string dataFileName,
string schemaFileName,
int maxSize)
: this(new StreamReader(dataFileName),
new StreamReader(schemaFileName),
maxSize)
{ }
public FileRowCollection(StreamReader dataReader,
StreamReader schemaReader)
: this(dataReader, schemaReader, Int32.MaxValue)
{ }
public FileRowCollection(StreamReader dataReader,
StreamReader schemaReader,
int maxSize)
: base(BlueDataUtility.createSchemaFromCSVSchemaFile(schemaReader), maxSize)
{
int id = 0;
while (dataReader.Peek() > 0)
{
try
{
addRow(BlueDataUtility.createRowFromCSVFileRow(dataReader.ReadLine(), mySchema, id++));
}
catch (Exception)
{
//BlueEventLog.logEvent (e.Message, BlueEventLevel.Warning);
}
}
}
}
}