diff --git a/src/DataStreamSuite/ArithmeticExpressions/GplexBuffers.cs b/src/DataStreamSuite/ArithmeticExpressions/GplexBuffers.cs
index 76dd35443..1bcc08e09 100644
--- a/src/DataStreamSuite/ArithmeticExpressions/GplexBuffers.cs
+++ b/src/DataStreamSuite/ArithmeticExpressions/GplexBuffers.cs
@@ -1,720 +1,720 @@
-// ==============================================================
-//
-// This code automatically produced from an embedded resource.
-// Do not edit this file, or it will become incompatible with
-// the specification from which it was generated.
-//
-// ==============================================================
-
-using System;
-using System.IO;
-using System.Text;
-using System.Collections.Generic;
-using System.Diagnostics.CodeAnalysis;
-using System.Runtime.Serialization;
-using System.Globalization;
-
-namespace QUT.GplexBuffers
-{
-// Code copied from GPLEX embedded resource
- [Serializable]
- internal class BufferException : Exception
- {
- public BufferException() { }
- public BufferException(string message) : base(message) { }
- public BufferException(string message, Exception innerException)
- : base(message, innerException) { }
- protected BufferException(SerializationInfo info, StreamingContext context)
- : base(info, context) { }
- }
-
- internal abstract class ScanBuff
- {
- private string fileNm;
-
- public const int EndOfFile = -1;
- public const int UnicodeReplacementChar = 0xFFFD;
-
- public bool IsFile { get { return (fileNm != null); } }
- public string FileName { get { return fileNm; } set { fileNm = value; } }
-
- public abstract int Pos { get; set; }
- public abstract int Read();
- public virtual void Mark() { }
-
- public abstract string GetString(int begin, int limit);
-
- public static ScanBuff GetBuffer(string source)
- {
- return new StringBuffer(source);
- }
-
- public static ScanBuff GetBuffer(IList source)
- {
- return new LineBuffer(source);
- }
-
-#if (!NOFILES)
- public static ScanBuff GetBuffer(Stream source)
- {
- return new BuildBuffer(source);
- }
-
-#if (!BYTEMODE)
- public static ScanBuff GetBuffer(Stream source, int fallbackCodePage)
- {
- return new BuildBuffer(source, fallbackCodePage);
- }
-#endif // !BYTEMODE
-#endif // !NOFILES
- }
-
- #region Buffer classes
-
- // ==============================================================
- // ===== Definitions for various ScanBuff derived classes ====
- // ==============================================================
- // =============== String input ================
- // ==============================================================
-
- ///
- /// This class reads characters from a single string as
- /// required, for example, by Visual Studio language services
- ///
- sealed class StringBuffer : ScanBuff
- {
- string str; // input buffer
- int bPos; // current position in buffer
- int sLen;
-
- public StringBuffer(string source)
- {
- this.str = source;
- this.sLen = source.Length;
- this.FileName = null;
- }
-
- public override int Read()
- {
- if (bPos < sLen) return str[bPos++];
- else if (bPos == sLen) { bPos++; return '\n'; } // one strike, see new line
- else { bPos++; return EndOfFile; } // two strikes and you're out!
- }
-
- public override string GetString(int begin, int limit)
- {
- // "limit" can be greater than sLen with the BABEL
- // option set. Read returns a "virtual" EOL if
- // an attempt is made to read past the end of the
- // string buffer. Without the guard any attempt
- // to fetch yytext for a token that includes the
- // EOL will throw an index exception.
- if (limit > sLen) limit = sLen;
- if (limit <= begin) return "";
- else return str.Substring(begin, limit - begin);
- }
-
- public override int Pos
- {
- get { return bPos; }
- set { bPos = value; }
- }
-
- public override string ToString() { return "StringBuffer"; }
- }
-
- // ==============================================================
- // The LineBuff class contributed by Nigel Horspool,
- // nigelh@cs.uvic.cs
- // ==============================================================
-
- sealed class LineBuffer : ScanBuff
- {
- IList line; // list of source lines from a file
- int numLines; // number of strings in line list
- string curLine; // current line in that list
- int cLine; // index of current line in the list
- int curLen; // length of current line
- int curLineStart; // position of line start in whole file
- int curLineEnd; // position of line end in whole file
- int maxPos; // max position ever visited in whole file
- int cPos; // ordinal number of code in source
-
- // Constructed from a list of strings, one per source line.
- // The lines have had trailing '\n' characters removed.
- public LineBuffer(IList lineList)
- {
- line = lineList;
- numLines = line.Count;
- cPos = curLineStart = 0;
- curLine = (numLines > 0 ? line[0] : "");
- maxPos = curLineEnd = curLen = curLine.Length;
- cLine = 1;
- FileName = null;
- }
-
- public override int Read()
- {
- if (cPos < curLineEnd)
- return curLine[cPos++ - curLineStart];
- if (cPos++ == curLineEnd)
- return '\n';
- if (cLine >= numLines)
- return EndOfFile;
- curLine = line[cLine];
- curLen = curLine.Length;
- curLineStart = curLineEnd + 1;
- curLineEnd = curLineStart + curLen;
- if (curLineEnd > maxPos)
- maxPos = curLineEnd;
- cLine++;
- return curLen > 0 ? curLine[0] : '\n';
- }
-
- // To speed up searches for the line containing a position
- private int cachedPosition;
- private int cachedIxdex;
- private int cachedLineStart;
-
- // Given a position pos within the entire source, the results are
- // ix -- the index of the containing line
- // lstart -- the position of the first character on that line
- private void findIndex(int pos, out int ix, out int lstart)
- {
- if (pos >= cachedPosition)
- {
- ix = cachedIxdex; lstart = cachedLineStart;
- }
- else
- {
- ix = lstart = 0;
- }
- while (ix < numLines)
- {
- int len = line[ix].Length + 1;
- if (pos < lstart + len) break;
- lstart += len;
- ix++;
- }
- cachedPosition = pos;
- cachedIxdex = ix;
- cachedLineStart = lstart;
- }
-
- public override string GetString(int begin, int limit)
- {
- if (begin >= maxPos || limit <= begin) return "";
- int endIx, begIx, endLineStart, begLineStart;
- findIndex(begin, out begIx, out begLineStart);
- int begCol = begin - begLineStart;
- findIndex(limit, out endIx, out endLineStart);
- int endCol = limit - endLineStart;
- string s = line[begIx];
- if (begIx == endIx)
- {
- // the usual case, substring all on one line
- return (endCol <= s.Length) ?
- s.Substring(begCol, endCol - begCol)
- : s.Substring(begCol) + "\n";
- }
- // the string spans multiple lines, yuk!
- StringBuilder sb = new StringBuilder();
- if (begCol < s.Length)
- sb.Append(s.Substring(begCol));
- for (; ; )
- {
- sb.Append("\n");
- s = line[++begIx];
- if (begIx >= endIx) break;
- sb.Append(s);
- }
- if (endCol <= s.Length)
- {
- sb.Append(s.Substring(0, endCol));
- }
- else
- {
- sb.Append(s);
- sb.Append("\n");
- }
- return sb.ToString();
- }
-
- public override int Pos
- {
- get { return cPos; }
- set
- {
- cPos = value;
- findIndex(cPos, out cLine, out curLineStart);
- // cLine should be the *next* line after curLine.
- curLine = (cLine < numLines ? line[cLine++] : "");
- curLineEnd = curLineStart + curLine.Length;
- }
- }
-
- public override string ToString() { return "LineBuffer"; }
- }
-
-#if (!NOFILES)
- // ==============================================================
- // ===== class BuildBuff : for unicode text files ========
- // ==============================================================
-
- class BuildBuffer : ScanBuff
- {
- // Double buffer for char stream.
- class BufferElement
- {
- StringBuilder bldr = new StringBuilder();
- StringBuilder next = new StringBuilder();
- int minIx;
- int maxIx;
- int brkIx;
- bool appendToNext;
-
- internal BufferElement() { }
-
- internal int MaxIndex { get { return maxIx; } }
- // internal int MinIndex { get { return minIx; } }
-
- internal char this[int index]
- {
- get
- {
- if (index < minIx || index >= maxIx)
- throw new BufferException("Index was outside data buffer");
- else if (index < brkIx)
- return bldr[index - minIx];
- else
- return next[index - brkIx];
- }
- }
-
- internal void Append(char[] block, int count)
- {
- maxIx += count;
- if (appendToNext)
- this.next.Append(block, 0, count);
- else
- {
- this.bldr.Append(block, 0, count);
- brkIx = maxIx;
- appendToNext = true;
- }
- }
-
- internal string GetString(int start, int limit)
- {
- if (limit <= start)
- return "";
- if (start >= minIx && limit <= maxIx)
- if (limit < brkIx) // String entirely in bldr builder
- return bldr.ToString(start - minIx, limit - start);
- else if (start >= brkIx) // String entirely in next builder
- return next.ToString(start - brkIx, limit - start);
- else // Must do a string-concatenation
- return
- bldr.ToString(start - minIx, brkIx - start) +
- next.ToString(0, limit - brkIx);
- else
- throw new BufferException("String was outside data buffer");
- }
-
- internal void Mark(int limit)
- {
- if (limit > brkIx + 16) // Rotate blocks
- {
- StringBuilder temp = bldr;
- bldr = next;
- next = temp;
- next.Length = 0;
- minIx = brkIx;
- brkIx = maxIx;
- }
- }
- }
-
- BufferElement data = new BufferElement();
-
- int bPos; // Postion index in the StringBuilder
- BlockReader NextBlk; // Delegate that serves char-arrays;
-
- private string EncodingName
- {
- get
- {
- StreamReader rdr = NextBlk.Target as StreamReader;
- return (rdr == null ? "raw-bytes" : rdr.CurrentEncoding.BodyName);
- }
- }
-
- public BuildBuffer(Stream stream)
- {
- FileStream fStrm = (stream as FileStream);
- if (fStrm != null) FileName = fStrm.Name;
- NextBlk = BlockReaderFactory.Raw(stream);
- }
-
-#if (!BYTEMODE)
- public BuildBuffer(Stream stream, int fallbackCodePage)
- {
- FileStream fStrm = (stream as FileStream);
- if (fStrm != null) FileName = fStrm.Name;
- NextBlk = BlockReaderFactory.Get(stream, fallbackCodePage);
- }
-#endif
-
- ///
- /// Marks a conservative lower bound for the buffer,
- /// allowing space to be reclaimed. If an application
- /// needs to call GetString at arbitrary past locations
- /// in the input stream, Mark() is not called.
- ///
- public override void Mark() { data.Mark(bPos - 2); }
-
- public override int Pos
- {
- get { return bPos; }
- set { bPos = value; }
- }
-
-
- ///
- /// Read returns the ordinal number of the next char, or
- /// EOF (-1) for an end of stream. Note that the next
- /// code point may require *two* calls of Read().
- ///
- ///
- public override int Read()
- {
- //
- // Characters at positions
- // [data.offset, data.offset + data.bldr.Length)
- // are available in data.bldr.
- //
- if (bPos < data.MaxIndex)
- {
- // ch0 cannot be EOF
- return (int)data[bPos++];
- }
- else // Read from underlying stream
- {
- // Experimental code, blocks of page size
- char[] chrs = new char[4096];
- int count = NextBlk(chrs, 0, 4096);
- if (count == 0)
- return EndOfFile;
- else
- {
- data.Append(chrs, count);
- return (int)data[bPos++];
- }
- }
- }
-
- public override string GetString(int begin, int limit)
- {
- return data.GetString(begin, limit);
- }
-
- public override string ToString()
- {
- return "StringBuilder buffer, encoding: " + this.EncodingName;
- }
- }
-
- // =============== End ScanBuff-derived classes ==================
-
- internal delegate int BlockReader(char[] block, int index, int number);
-
- // A delegate factory, serving up a delegate that
- // reads a block of characters from the underlying
- // encoded stream, via a StreamReader object.
- //
- internal static class BlockReaderFactory
- {
- public static BlockReader Raw(Stream stream)
- {
- return delegate(char[] block, int index, int number)
- {
- byte[] b = new byte[number];
- int count = stream.Read(b, 0, number);
- int i = 0;
- int j = index;
- for (; i < count; i++, j++)
- block[j] = (char)b[i];
- return count;
- };
- }
-
-#if (!BYTEMODE)
- public static BlockReader Get(Stream stream, int fallbackCodePage)
- {
- Encoding encoding;
- int preamble = Preamble(stream);
-
- if (preamble != 0) // There is a valid BOM here!
- encoding = Encoding.GetEncoding(preamble);
- else if (fallbackCodePage == -1) // Fallback is "raw" bytes
- return Raw(stream);
- else if (fallbackCodePage != -2) // Anything but "guess"
- encoding = Encoding.GetEncoding(fallbackCodePage);
- else // This is the "guess" option
- {
- int guess = new Guesser(stream).GuessCodePage();
- stream.Seek(0, SeekOrigin.Begin);
- if (guess == -1) // ==> this is a 7-bit file
- encoding = Encoding.ASCII;
- else if (guess == 65001)
- encoding = Encoding.UTF8;
- else // ==> use the machine default
- encoding = Encoding.Default;
- }
- StreamReader reader = new StreamReader(stream, encoding);
- return reader.Read;
- }
-
- static int Preamble(Stream stream)
- {
- int b0 = stream.ReadByte();
- int b1 = stream.ReadByte();
-
- if (b0 == 0xfe && b1 == 0xff)
- return 1201; // UTF16BE
- if (b0 == 0xff && b1 == 0xfe)
- return 1200; // UTF16LE
-
- int b2 = stream.ReadByte();
- if (b0 == 0xef && b1 == 0xbb && b2 == 0xbf)
- return 65001; // UTF8
- //
- // There is no unicode preamble, so we
- // return denoter for the machine default.
- //
- stream.Seek(0, SeekOrigin.Begin);
- return 0;
- }
-#endif // !BYTEMODE
- }
-#endif // !NOFILES
- #endregion Buffer classes
-
- // ==============================================================
- // ============ class CodePageHandling =============
- // ==============================================================
-#if (!NOFILES)
- public static class CodePageHandling
- {
- public static int GetCodePage(string option)
- {
- string command = option.ToUpperInvariant();
- if (command.StartsWith("CodePage:", StringComparison.OrdinalIgnoreCase))
- command = command.Substring(9);
- try
- {
- if (command.Equals("RAW"))
- return -1;
- else if (command.Equals("GUESS"))
- return -2;
- else if (command.Equals("DEFAULT"))
- return 0;
- else if (char.IsDigit(command[0]))
- return int.Parse(command, CultureInfo.InvariantCulture);
- else
- {
- Encoding enc = Encoding.GetEncoding(command);
- return enc.CodePage;
- }
- }
- catch (FormatException)
- {
- Console.Error.WriteLine(
- "Invalid format \"{0}\", using machine default", option);
- }
- catch (ArgumentException)
- {
- Console.Error.WriteLine(
- "Unknown code page \"{0}\", using machine default", option);
- }
- return 0;
- }
- }
-#region guesser
-#if (!BYTEMODE)
- // ==============================================================
- // ============ Encoding Guesser =============
- // ==============================================================
-
- ///
- /// This class provides a simple finite state automaton that
- /// scans the file looking for (1) valid UTF-8 byte patterns,
- /// (2) bytes >= 0x80 which are not part of a UTF-8 sequence.
- /// The method then guesses whether it is UTF-8 or maybe some
- /// local machine default encoding. This works well for the
- /// various Latin encodings.
- ///
- internal class Guesser
- {
- ScanBuff buffer;
-
- public int GuessCodePage() { return Scan(); }
-
- const int maxAccept = 10;
- const int initial = 0;
- const int eofNum = 0;
- const int goStart = -1;
- const int INITIAL = 0;
- const int EndToken = 0;
-
- #region user code
- /*
- * Reads the bytes of a file to determine if it is
- * UTF-8 or a single-byte code page file.
- */
- public long utfX;
- public long uppr;
- #endregion user code
-
- int state;
- int currentStart = startState[0];
- int code;
-
- #region ScannerTables
- static int[] startState = new int[] { 11, 0 };
-
- #region CharacterMap
- static sbyte[] map = new sbyte[256] {
-/* '\0' */ 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0,
-/* '\x10' */ 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0,
-/* '\x20' */ 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0,
-/* '0' */ 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0,
-/* '@' */ 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0,
-/* 'P' */ 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0,
-/* '`' */ 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0,
-/* 'p' */ 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0,
-/* '\x80' */ 2, 2, 2, 2, 2, 2, 2, 2, 2, 2, 2, 2, 2, 2, 2, 2,
-/* '\x90' */ 2, 2, 2, 2, 2, 2, 2, 2, 2, 2, 2, 2, 2, 2, 2, 2,
-/* '\xA0' */ 2, 2, 2, 2, 2, 2, 2, 2, 2, 2, 2, 2, 2, 2, 2, 2,
-/* '\xB0' */ 2, 2, 2, 2, 2, 2, 2, 2, 2, 2, 2, 2, 2, 2, 2, 2,
-/* '\xC0' */ 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1,
-/* '\xD0' */ 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1,
-/* '\xE0' */ 3, 3, 3, 3, 3, 3, 3, 3, 3, 3, 3, 3, 3, 3, 3, 3,
-/* '\xF0' */ 4, 4, 4, 4, 4, 4, 4, 4, 5, 5, 5, 5, 5, 5, 5, 5 };
- #endregion
-
- static sbyte[][] nextState = new sbyte[][] {
- new sbyte[] {0, 0, 0, 0, 0, 0},
- new sbyte[] {-1, -1, 10, -1, -1, -1},
- new sbyte[] {-1, -1, -1, -1, -1, -1},
- new sbyte[] {-1, -1, 8, -1, -1, -1},
- new sbyte[] {-1, -1, 5, -1, -1, -1},
- new sbyte[] {-1, -1, 6, -1, -1, -1},
- new sbyte[] {-1, -1, 7, -1, -1, -1},
- null,
- new sbyte[] {-1, -1, 9, -1, -1, -1},
- null,
- null,
- new sbyte[] {-1, 1, 2, 3, 4, 2}
- };
-
-
- [SuppressMessage("Microsoft.Performance", "CA1810:InitializeReferenceTypeStaticFieldsInline")]
- // Reason for suppression: cannot have self-reference in array initializer.
- static Guesser()
- {
- nextState[7] = nextState[2];
- nextState[9] = nextState[2];
- nextState[10] = nextState[2];
- }
-
- int NextState()
- {
- if (code == ScanBuff.EndOfFile)
- return eofNum;
- else
- return nextState[state][map[code]];
- }
- #endregion
-
- public Guesser(System.IO.Stream file) { SetSource(file); }
-
- public void SetSource(System.IO.Stream source)
- {
- this.buffer = new BuildBuffer(source);
- code = buffer.Read();
- }
-
- int Scan()
- {
- for (; ; )
- {
- int next;
- state = currentStart;
- while ((next = NextState()) == goStart)
- code = buffer.Read();
-
- state = next;
- code = buffer.Read();
-
- while ((next = NextState()) > eofNum)
- {
- state = next;
- code = buffer.Read();
- }
- if (state <= maxAccept)
- {
- #region ActionSwitch
-#pragma warning disable 162
- switch (state)
- {
- case eofNum:
- switch (currentStart)
- {
- case 11:
- if (utfX == 0 && uppr == 0) return -1; /* raw ascii */
- else if (uppr * 10 > utfX) return 0; /* default code page */
- else return 65001; /* UTF-8 encoding */
- break;
- }
- return EndToken;
- case 1: // Recognized '{Upper128}', Shortest string "\xC0"
- case 2: // Recognized '{Upper128}', Shortest string "\x80"
- case 3: // Recognized '{Upper128}', Shortest string "\xE0"
- case 4: // Recognized '{Upper128}', Shortest string "\xF0"
- uppr++;
- break;
- case 5: // Recognized '{Utf8pfx4}{Utf8cont}', Shortest string "\xF0\x80"
- uppr += 2;
- break;
- case 6: // Recognized '{Utf8pfx4}{Utf8cont}{2}', Shortest string "\xF0\x80\x80"
- uppr += 3;
- break;
- case 7: // Recognized '{Utf8pfx4}{Utf8cont}{3}', Shortest string "\xF0\x80\x80\x80"
- utfX += 3;
- break;
- case 8: // Recognized '{Utf8pfx3}{Utf8cont}', Shortest string "\xE0\x80"
- uppr += 2;
- break;
- case 9: // Recognized '{Utf8pfx3}{Utf8cont}{2}', Shortest string "\xE0\x80\x80"
- utfX += 2;
- break;
- case 10: // Recognized '{Utf8pfx2}{Utf8cont}', Shortest string "\xC0\x80"
- utfX++;
- break;
- default:
- break;
- }
-#pragma warning restore 162
- #endregion
- }
- }
- }
- } // end class Guesser
-
-#endif // !BYTEMODE
-#endregion
-#endif // !NOFILES
-
-// End of code copied from embedded resource
-}
+// ==============================================================
+//
+// This code automatically produced from an embedded resource.
+// Do not edit this file, or it will become incompatible with
+// the specification from which it was generated.
+//
+// ==============================================================
+
+using System;
+using System.IO;
+using System.Text;
+using System.Collections.Generic;
+using System.Diagnostics.CodeAnalysis;
+using System.Runtime.Serialization;
+using System.Globalization;
+
+namespace QUT.GplexBuffers
+{
+// Code copied from GPLEX embedded resource
+ [Serializable]
+ internal class BufferException : Exception
+ {
+ public BufferException() { }
+ public BufferException(string message) : base(message) { }
+ public BufferException(string message, Exception innerException)
+ : base(message, innerException) { }
+ protected BufferException(SerializationInfo info, StreamingContext context)
+ : base(info, context) { }
+ }
+
+ internal abstract class ScanBuff
+ {
+ private string fileNm;
+
+ public const int EndOfFile = -1;
+ public const int UnicodeReplacementChar = 0xFFFD;
+
+ public bool IsFile { get { return (fileNm != null); } }
+ public string FileName { get { return fileNm; } set { fileNm = value; } }
+
+ public abstract int Pos { get; set; }
+ public abstract int Read();
+ public virtual void Mark() { }
+
+ public abstract string GetString(int begin, int limit);
+
+ public static ScanBuff GetBuffer(string source)
+ {
+ return new StringBuffer(source);
+ }
+
+ public static ScanBuff GetBuffer(IList source)
+ {
+ return new LineBuffer(source);
+ }
+
+#if (!NOFILES)
+ public static ScanBuff GetBuffer(Stream source)
+ {
+ return new BuildBuffer(source);
+ }
+
+#if (!BYTEMODE)
+ public static ScanBuff GetBuffer(Stream source, int fallbackCodePage)
+ {
+ return new BuildBuffer(source, fallbackCodePage);
+ }
+#endif // !BYTEMODE
+#endif // !NOFILES
+ }
+
+ #region Buffer classes
+
+ // ==============================================================
+ // ===== Definitions for various ScanBuff derived classes ====
+ // ==============================================================
+ // =============== String input ================
+ // ==============================================================
+
+ ///
+ /// This class reads characters from a single string as
+ /// required, for example, by Visual Studio language services
+ ///
+ sealed class StringBuffer : ScanBuff
+ {
+ string str; // input buffer
+ int bPos; // current position in buffer
+ int sLen;
+
+ public StringBuffer(string source)
+ {
+ this.str = source;
+ this.sLen = source.Length;
+ this.FileName = null;
+ }
+
+ public override int Read()
+ {
+ if (bPos < sLen) return str[bPos++];
+ else if (bPos == sLen) { bPos++; return '\n'; } // one strike, see new line
+ else { bPos++; return EndOfFile; } // two strikes and you're out!
+ }
+
+ public override string GetString(int begin, int limit)
+ {
+ // "limit" can be greater than sLen with the BABEL
+ // option set. Read returns a "virtual" EOL if
+ // an attempt is made to read past the end of the
+ // string buffer. Without the guard any attempt
+ // to fetch yytext for a token that includes the
+ // EOL will throw an index exception.
+ if (limit > sLen) limit = sLen;
+ if (limit <= begin) return "";
+ else return str.Substring(begin, limit - begin);
+ }
+
+ public override int Pos
+ {
+ get { return bPos; }
+ set { bPos = value; }
+ }
+
+ public override string ToString() { return "StringBuffer"; }
+ }
+
+ // ==============================================================
+ // The LineBuff class contributed by Nigel Horspool,
+ // nigelh@cs.uvic.cs
+ // ==============================================================
+
+ sealed class LineBuffer : ScanBuff
+ {
+ IList line; // list of source lines from a file
+ int numLines; // number of strings in line list
+ string curLine; // current line in that list
+ int cLine; // index of current line in the list
+ int curLen; // length of current line
+ int curLineStart; // position of line start in whole file
+ int curLineEnd; // position of line end in whole file
+ int maxPos; // max position ever visited in whole file
+ int cPos; // ordinal number of code in source
+
+ // Constructed from a list of strings, one per source line.
+ // The lines have had trailing '\n' characters removed.
+ public LineBuffer(IList lineList)
+ {
+ line = lineList;
+ numLines = line.Count;
+ cPos = curLineStart = 0;
+ curLine = (numLines > 0 ? line[0] : "");
+ maxPos = curLineEnd = curLen = curLine.Length;
+ cLine = 1;
+ FileName = null;
+ }
+
+ public override int Read()
+ {
+ if (cPos < curLineEnd)
+ return curLine[cPos++ - curLineStart];
+ if (cPos++ == curLineEnd)
+ return '\n';
+ if (cLine >= numLines)
+ return EndOfFile;
+ curLine = line[cLine];
+ curLen = curLine.Length;
+ curLineStart = curLineEnd + 1;
+ curLineEnd = curLineStart + curLen;
+ if (curLineEnd > maxPos)
+ maxPos = curLineEnd;
+ cLine++;
+ return curLen > 0 ? curLine[0] : '\n';
+ }
+
+ // To speed up searches for the line containing a position
+ private int cachedPosition;
+ private int cachedIxdex;
+ private int cachedLineStart;
+
+ // Given a position pos within the entire source, the results are
+ // ix -- the index of the containing line
+ // lstart -- the position of the first character on that line
+ private void findIndex(int pos, out int ix, out int lstart)
+ {
+ if (pos >= cachedPosition)
+ {
+ ix = cachedIxdex; lstart = cachedLineStart;
+ }
+ else
+ {
+ ix = lstart = 0;
+ }
+ while (ix < numLines)
+ {
+ int len = line[ix].Length + 1;
+ if (pos < lstart + len) break;
+ lstart += len;
+ ix++;
+ }
+ cachedPosition = pos;
+ cachedIxdex = ix;
+ cachedLineStart = lstart;
+ }
+
+ public override string GetString(int begin, int limit)
+ {
+ if (begin >= maxPos || limit <= begin) return "";
+ int endIx, begIx, endLineStart, begLineStart;
+ findIndex(begin, out begIx, out begLineStart);
+ int begCol = begin - begLineStart;
+ findIndex(limit, out endIx, out endLineStart);
+ int endCol = limit - endLineStart;
+ string s = line[begIx];
+ if (begIx == endIx)
+ {
+ // the usual case, substring all on one line
+ return (endCol <= s.Length) ?
+ s.Substring(begCol, endCol - begCol)
+ : s.Substring(begCol) + "\n";
+ }
+ // the string spans multiple lines, yuk!
+ StringBuilder sb = new StringBuilder();
+ if (begCol < s.Length)
+ sb.Append(s.Substring(begCol));
+ for (; ; )
+ {
+ sb.Append("\n");
+ s = line[++begIx];
+ if (begIx >= endIx) break;
+ sb.Append(s);
+ }
+ if (endCol <= s.Length)
+ {
+ sb.Append(s.Substring(0, endCol));
+ }
+ else
+ {
+ sb.Append(s);
+ sb.Append("\n");
+ }
+ return sb.ToString();
+ }
+
+ public override int Pos
+ {
+ get { return cPos; }
+ set
+ {
+ cPos = value;
+ findIndex(cPos, out cLine, out curLineStart);
+ // cLine should be the *next* line after curLine.
+ curLine = (cLine < numLines ? line[cLine++] : "");
+ curLineEnd = curLineStart + curLine.Length;
+ }
+ }
+
+ public override string ToString() { return "LineBuffer"; }
+ }
+
+#if (!NOFILES)
+ // ==============================================================
+ // ===== class BuildBuff : for unicode text files ========
+ // ==============================================================
+
+ class BuildBuffer : ScanBuff
+ {
+ // Double buffer for char stream.
+ class BufferElement
+ {
+ StringBuilder bldr = new StringBuilder();
+ StringBuilder next = new StringBuilder();
+ int minIx;
+ int maxIx;
+ int brkIx;
+ bool appendToNext;
+
+ internal BufferElement() { }
+
+ internal int MaxIndex { get { return maxIx; } }
+ // internal int MinIndex { get { return minIx; } }
+
+ internal char this[int index]
+ {
+ get
+ {
+ if (index < minIx || index >= maxIx)
+ throw new BufferException("Index was outside data buffer");
+ else if (index < brkIx)
+ return bldr[index - minIx];
+ else
+ return next[index - brkIx];
+ }
+ }
+
+ internal void Append(char[] block, int count)
+ {
+ maxIx += count;
+ if (appendToNext)
+ this.next.Append(block, 0, count);
+ else
+ {
+ this.bldr.Append(block, 0, count);
+ brkIx = maxIx;
+ appendToNext = true;
+ }
+ }
+
+ internal string GetString(int start, int limit)
+ {
+ if (limit <= start)
+ return "";
+ if (start >= minIx && limit <= maxIx)
+ if (limit < brkIx) // String entirely in bldr builder
+ return bldr.ToString(start - minIx, limit - start);
+ else if (start >= brkIx) // String entirely in next builder
+ return next.ToString(start - brkIx, limit - start);
+ else // Must do a string-concatenation
+ return
+ bldr.ToString(start - minIx, brkIx - start) +
+ next.ToString(0, limit - brkIx);
+ else
+ throw new BufferException("String was outside data buffer");
+ }
+
+ internal void Mark(int limit)
+ {
+ if (limit > brkIx + 16) // Rotate blocks
+ {
+ StringBuilder temp = bldr;
+ bldr = next;
+ next = temp;
+ next.Length = 0;
+ minIx = brkIx;
+ brkIx = maxIx;
+ }
+ }
+ }
+
+ BufferElement data = new BufferElement();
+
+ int bPos; // Postion index in the StringBuilder
+ BlockReader NextBlk; // Delegate that serves char-arrays;
+
+ private string EncodingName
+ {
+ get
+ {
+ StreamReader rdr = NextBlk.Target as StreamReader;
+ return (rdr == null ? "raw-bytes" : rdr.CurrentEncoding.BodyName);
+ }
+ }
+
+ public BuildBuffer(Stream stream)
+ {
+ FileStream fStrm = (stream as FileStream);
+ if (fStrm != null) FileName = fStrm.Name;
+ NextBlk = BlockReaderFactory.Raw(stream);
+ }
+
+#if (!BYTEMODE)
+ public BuildBuffer(Stream stream, int fallbackCodePage)
+ {
+ FileStream fStrm = (stream as FileStream);
+ if (fStrm != null) FileName = fStrm.Name;
+ NextBlk = BlockReaderFactory.Get(stream, fallbackCodePage);
+ }
+#endif
+
+ ///
+ /// Marks a conservative lower bound for the buffer,
+ /// allowing space to be reclaimed. If an application
+ /// needs to call GetString at arbitrary past locations
+ /// in the input stream, Mark() is not called.
+ ///
+ public override void Mark() { data.Mark(bPos - 2); }
+
+ public override int Pos
+ {
+ get { return bPos; }
+ set { bPos = value; }
+ }
+
+
+ ///
+ /// Read returns the ordinal number of the next char, or
+ /// EOF (-1) for an end of stream. Note that the next
+ /// code point may require *two* calls of Read().
+ ///
+ ///
+ public override int Read()
+ {
+ //
+ // Characters at positions
+ // [data.offset, data.offset + data.bldr.Length)
+ // are available in data.bldr.
+ //
+ if (bPos < data.MaxIndex)
+ {
+ // ch0 cannot be EOF
+ return (int)data[bPos++];
+ }
+ else // Read from underlying stream
+ {
+ // Experimental code, blocks of page size
+ char[] chrs = new char[4096];
+ int count = NextBlk(chrs, 0, 4096);
+ if (count == 0)
+ return EndOfFile;
+ else
+ {
+ data.Append(chrs, count);
+ return (int)data[bPos++];
+ }
+ }
+ }
+
+ public override string GetString(int begin, int limit)
+ {
+ return data.GetString(begin, limit);
+ }
+
+ public override string ToString()
+ {
+ return "StringBuilder buffer, encoding: " + this.EncodingName;
+ }
+ }
+
+ // =============== End ScanBuff-derived classes ==================
+
+ internal delegate int BlockReader(char[] block, int index, int number);
+
+ // A delegate factory, serving up a delegate that
+ // reads a block of characters from the underlying
+ // encoded stream, via a StreamReader object.
+ //
+ internal static class BlockReaderFactory
+ {
+ public static BlockReader Raw(Stream stream)
+ {
+ return delegate(char[] block, int index, int number)
+ {
+ byte[] b = new byte[number];
+ int count = stream.Read(b, 0, number);
+ int i = 0;
+ int j = index;
+ for (; i < count; i++, j++)
+ block[j] = (char)b[i];
+ return count;
+ };
+ }
+
+#if (!BYTEMODE)
+ public static BlockReader Get(Stream stream, int fallbackCodePage)
+ {
+ Encoding encoding;
+ int preamble = Preamble(stream);
+
+ if (preamble != 0) // There is a valid BOM here!
+ encoding = Encoding.GetEncoding(preamble);
+ else if (fallbackCodePage == -1) // Fallback is "raw" bytes
+ return Raw(stream);
+ else if (fallbackCodePage != -2) // Anything but "guess"
+ encoding = Encoding.GetEncoding(fallbackCodePage);
+ else // This is the "guess" option
+ {
+ int guess = new Guesser(stream).GuessCodePage();
+ stream.Seek(0, SeekOrigin.Begin);
+ if (guess == -1) // ==> this is a 7-bit file
+ encoding = Encoding.ASCII;
+ else if (guess == 65001)
+ encoding = Encoding.UTF8;
+ else // ==> use the machine default
+ encoding = Encoding.Default;
+ }
+ StreamReader reader = new StreamReader(stream, encoding);
+ return reader.Read;
+ }
+
+ static int Preamble(Stream stream)
+ {
+ int b0 = stream.ReadByte();
+ int b1 = stream.ReadByte();
+
+ if (b0 == 0xfe && b1 == 0xff)
+ return 1201; // UTF16BE
+ if (b0 == 0xff && b1 == 0xfe)
+ return 1200; // UTF16LE
+
+ int b2 = stream.ReadByte();
+ if (b0 == 0xef && b1 == 0xbb && b2 == 0xbf)
+ return 65001; // UTF8
+ //
+ // There is no unicode preamble, so we
+ // return denoter for the machine default.
+ //
+ stream.Seek(0, SeekOrigin.Begin);
+ return 0;
+ }
+#endif // !BYTEMODE
+ }
+#endif // !NOFILES
+ #endregion Buffer classes
+
+ // ==============================================================
+ // ============ class CodePageHandling =============
+ // ==============================================================
+#if (!NOFILES)
+ public static class CodePageHandling
+ {
+ public static int GetCodePage(string option)
+ {
+ string command = option.ToUpperInvariant();
+ if (command.StartsWith("CodePage:", StringComparison.OrdinalIgnoreCase))
+ command = command.Substring(9);
+ try
+ {
+ if (command.Equals("RAW"))
+ return -1;
+ else if (command.Equals("GUESS"))
+ return -2;
+ else if (command.Equals("DEFAULT"))
+ return 0;
+ else if (char.IsDigit(command[0]))
+ return int.Parse(command, CultureInfo.InvariantCulture);
+ else
+ {
+ Encoding enc = Encoding.GetEncoding(command);
+ return enc.CodePage;
+ }
+ }
+ catch (FormatException)
+ {
+ Console.Error.WriteLine(
+ "Invalid format \"{0}\", using machine default", option);
+ }
+ catch (ArgumentException)
+ {
+ Console.Error.WriteLine(
+ "Unknown code page \"{0}\", using machine default", option);
+ }
+ return 0;
+ }
+ }
+#region guesser
+#if (!BYTEMODE)
+ // ==============================================================
+ // ============ Encoding Guesser =============
+ // ==============================================================
+
+ ///
+ /// This class provides a simple finite state automaton that
+ /// scans the file looking for (1) valid UTF-8 byte patterns,
+ /// (2) bytes >= 0x80 which are not part of a UTF-8 sequence.
+ /// The method then guesses whether it is UTF-8 or maybe some
+ /// local machine default encoding. This works well for the
+ /// various Latin encodings.
+ ///
+ internal class Guesser
+ {
+ ScanBuff buffer;
+
+ public int GuessCodePage() { return Scan(); }
+
+ const int maxAccept = 10;
+ const int initial = 0;
+ const int eofNum = 0;
+ const int goStart = -1;
+ const int INITIAL = 0;
+ const int EndToken = 0;
+
+ #region user code
+ /*
+ * Reads the bytes of a file to determine if it is
+ * UTF-8 or a single-byte code page file.
+ */
+ public long utfX;
+ public long uppr;
+ #endregion user code
+
+ int state;
+ int currentStart = startState[0];
+ int code;
+
+ #region ScannerTables
+ static int[] startState = new int[] { 11, 0 };
+
+ #region CharacterMap
+ static sbyte[] map = new sbyte[256] {
+/* '\0' */ 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0,
+/* '\x10' */ 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0,
+/* '\x20' */ 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0,
+/* '0' */ 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0,
+/* '@' */ 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0,
+/* 'P' */ 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0,
+/* '`' */ 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0,
+/* 'p' */ 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0,
+/* '\x80' */ 2, 2, 2, 2, 2, 2, 2, 2, 2, 2, 2, 2, 2, 2, 2, 2,
+/* '\x90' */ 2, 2, 2, 2, 2, 2, 2, 2, 2, 2, 2, 2, 2, 2, 2, 2,
+/* '\xA0' */ 2, 2, 2, 2, 2, 2, 2, 2, 2, 2, 2, 2, 2, 2, 2, 2,
+/* '\xB0' */ 2, 2, 2, 2, 2, 2, 2, 2, 2, 2, 2, 2, 2, 2, 2, 2,
+/* '\xC0' */ 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1,
+/* '\xD0' */ 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1,
+/* '\xE0' */ 3, 3, 3, 3, 3, 3, 3, 3, 3, 3, 3, 3, 3, 3, 3, 3,
+/* '\xF0' */ 4, 4, 4, 4, 4, 4, 4, 4, 5, 5, 5, 5, 5, 5, 5, 5 };
+ #endregion
+
+ static sbyte[][] nextState = new sbyte[][] {
+ new sbyte[] {0, 0, 0, 0, 0, 0},
+ new sbyte[] {-1, -1, 10, -1, -1, -1},
+ new sbyte[] {-1, -1, -1, -1, -1, -1},
+ new sbyte[] {-1, -1, 8, -1, -1, -1},
+ new sbyte[] {-1, -1, 5, -1, -1, -1},
+ new sbyte[] {-1, -1, 6, -1, -1, -1},
+ new sbyte[] {-1, -1, 7, -1, -1, -1},
+ null,
+ new sbyte[] {-1, -1, 9, -1, -1, -1},
+ null,
+ null,
+ new sbyte[] {-1, 1, 2, 3, 4, 2}
+ };
+
+
+ [SuppressMessage("Microsoft.Performance", "CA1810:InitializeReferenceTypeStaticFieldsInline")]
+ // Reason for suppression: cannot have self-reference in array initializer.
+ static Guesser()
+ {
+ nextState[7] = nextState[2];
+ nextState[9] = nextState[2];
+ nextState[10] = nextState[2];
+ }
+
+ int NextState()
+ {
+ if (code == ScanBuff.EndOfFile)
+ return eofNum;
+ else
+ return nextState[state][map[code]];
+ }
+ #endregion
+
+ public Guesser(System.IO.Stream file) { SetSource(file); }
+
+ public void SetSource(System.IO.Stream source)
+ {
+ this.buffer = new BuildBuffer(source);
+ code = buffer.Read();
+ }
+
+ int Scan()
+ {
+ for (; ; )
+ {
+ int next;
+ state = currentStart;
+ while ((next = NextState()) == goStart)
+ code = buffer.Read();
+
+ state = next;
+ code = buffer.Read();
+
+ while ((next = NextState()) > eofNum)
+ {
+ state = next;
+ code = buffer.Read();
+ }
+ if (state <= maxAccept)
+ {
+ #region ActionSwitch
+#pragma warning disable 162
+ switch (state)
+ {
+ case eofNum:
+ switch (currentStart)
+ {
+ case 11:
+ if (utfX == 0 && uppr == 0) return -1; /* raw ascii */
+ else if (uppr * 10 > utfX) return 0; /* default code page */
+ else return 65001; /* UTF-8 encoding */
+ break;
+ }
+ return EndToken;
+ case 1: // Recognized '{Upper128}', Shortest string "\xC0"
+ case 2: // Recognized '{Upper128}', Shortest string "\x80"
+ case 3: // Recognized '{Upper128}', Shortest string "\xE0"
+ case 4: // Recognized '{Upper128}', Shortest string "\xF0"
+ uppr++;
+ break;
+ case 5: // Recognized '{Utf8pfx4}{Utf8cont}', Shortest string "\xF0\x80"
+ uppr += 2;
+ break;
+ case 6: // Recognized '{Utf8pfx4}{Utf8cont}{2}', Shortest string "\xF0\x80\x80"
+ uppr += 3;
+ break;
+ case 7: // Recognized '{Utf8pfx4}{Utf8cont}{3}', Shortest string "\xF0\x80\x80\x80"
+ utfX += 3;
+ break;
+ case 8: // Recognized '{Utf8pfx3}{Utf8cont}', Shortest string "\xE0\x80"
+ uppr += 2;
+ break;
+ case 9: // Recognized '{Utf8pfx3}{Utf8cont}{2}', Shortest string "\xE0\x80\x80"
+ utfX += 2;
+ break;
+ case 10: // Recognized '{Utf8pfx2}{Utf8cont}', Shortest string "\xC0\x80"
+ utfX++;
+ break;
+ default:
+ break;
+ }
+#pragma warning restore 162
+ #endregion
+ }
+ }
+ }
+ } // end class Guesser
+
+#endif // !BYTEMODE
+#endregion
+#endif // !NOFILES
+
+// End of code copied from embedded resource
+}
diff --git a/src/DataStreamSuite/BooleanExpressions/GplexBuffers.cs b/src/DataStreamSuite/BooleanExpressions/GplexBuffers.cs
index 76dd35443..1bcc08e09 100644
--- a/src/DataStreamSuite/BooleanExpressions/GplexBuffers.cs
+++ b/src/DataStreamSuite/BooleanExpressions/GplexBuffers.cs
@@ -1,720 +1,720 @@
-// ==============================================================
-//
-// This code automatically produced from an embedded resource.
-// Do not edit this file, or it will become incompatible with
-// the specification from which it was generated.
-//
-// ==============================================================
-
-using System;
-using System.IO;
-using System.Text;
-using System.Collections.Generic;
-using System.Diagnostics.CodeAnalysis;
-using System.Runtime.Serialization;
-using System.Globalization;
-
-namespace QUT.GplexBuffers
-{
-// Code copied from GPLEX embedded resource
- [Serializable]
- internal class BufferException : Exception
- {
- public BufferException() { }
- public BufferException(string message) : base(message) { }
- public BufferException(string message, Exception innerException)
- : base(message, innerException) { }
- protected BufferException(SerializationInfo info, StreamingContext context)
- : base(info, context) { }
- }
-
- internal abstract class ScanBuff
- {
- private string fileNm;
-
- public const int EndOfFile = -1;
- public const int UnicodeReplacementChar = 0xFFFD;
-
- public bool IsFile { get { return (fileNm != null); } }
- public string FileName { get { return fileNm; } set { fileNm = value; } }
-
- public abstract int Pos { get; set; }
- public abstract int Read();
- public virtual void Mark() { }
-
- public abstract string GetString(int begin, int limit);
-
- public static ScanBuff GetBuffer(string source)
- {
- return new StringBuffer(source);
- }
-
- public static ScanBuff GetBuffer(IList source)
- {
- return new LineBuffer(source);
- }
-
-#if (!NOFILES)
- public static ScanBuff GetBuffer(Stream source)
- {
- return new BuildBuffer(source);
- }
-
-#if (!BYTEMODE)
- public static ScanBuff GetBuffer(Stream source, int fallbackCodePage)
- {
- return new BuildBuffer(source, fallbackCodePage);
- }
-#endif // !BYTEMODE
-#endif // !NOFILES
- }
-
- #region Buffer classes
-
- // ==============================================================
- // ===== Definitions for various ScanBuff derived classes ====
- // ==============================================================
- // =============== String input ================
- // ==============================================================
-
- ///
- /// This class reads characters from a single string as
- /// required, for example, by Visual Studio language services
- ///
- sealed class StringBuffer : ScanBuff
- {
- string str; // input buffer
- int bPos; // current position in buffer
- int sLen;
-
- public StringBuffer(string source)
- {
- this.str = source;
- this.sLen = source.Length;
- this.FileName = null;
- }
-
- public override int Read()
- {
- if (bPos < sLen) return str[bPos++];
- else if (bPos == sLen) { bPos++; return '\n'; } // one strike, see new line
- else { bPos++; return EndOfFile; } // two strikes and you're out!
- }
-
- public override string GetString(int begin, int limit)
- {
- // "limit" can be greater than sLen with the BABEL
- // option set. Read returns a "virtual" EOL if
- // an attempt is made to read past the end of the
- // string buffer. Without the guard any attempt
- // to fetch yytext for a token that includes the
- // EOL will throw an index exception.
- if (limit > sLen) limit = sLen;
- if (limit <= begin) return "";
- else return str.Substring(begin, limit - begin);
- }
-
- public override int Pos
- {
- get { return bPos; }
- set { bPos = value; }
- }
-
- public override string ToString() { return "StringBuffer"; }
- }
-
- // ==============================================================
- // The LineBuff class contributed by Nigel Horspool,
- // nigelh@cs.uvic.cs
- // ==============================================================
-
- sealed class LineBuffer : ScanBuff
- {
- IList line; // list of source lines from a file
- int numLines; // number of strings in line list
- string curLine; // current line in that list
- int cLine; // index of current line in the list
- int curLen; // length of current line
- int curLineStart; // position of line start in whole file
- int curLineEnd; // position of line end in whole file
- int maxPos; // max position ever visited in whole file
- int cPos; // ordinal number of code in source
-
- // Constructed from a list of strings, one per source line.
- // The lines have had trailing '\n' characters removed.
- public LineBuffer(IList lineList)
- {
- line = lineList;
- numLines = line.Count;
- cPos = curLineStart = 0;
- curLine = (numLines > 0 ? line[0] : "");
- maxPos = curLineEnd = curLen = curLine.Length;
- cLine = 1;
- FileName = null;
- }
-
- public override int Read()
- {
- if (cPos < curLineEnd)
- return curLine[cPos++ - curLineStart];
- if (cPos++ == curLineEnd)
- return '\n';
- if (cLine >= numLines)
- return EndOfFile;
- curLine = line[cLine];
- curLen = curLine.Length;
- curLineStart = curLineEnd + 1;
- curLineEnd = curLineStart + curLen;
- if (curLineEnd > maxPos)
- maxPos = curLineEnd;
- cLine++;
- return curLen > 0 ? curLine[0] : '\n';
- }
-
- // To speed up searches for the line containing a position
- private int cachedPosition;
- private int cachedIxdex;
- private int cachedLineStart;
-
- // Given a position pos within the entire source, the results are
- // ix -- the index of the containing line
- // lstart -- the position of the first character on that line
- private void findIndex(int pos, out int ix, out int lstart)
- {
- if (pos >= cachedPosition)
- {
- ix = cachedIxdex; lstart = cachedLineStart;
- }
- else
- {
- ix = lstart = 0;
- }
- while (ix < numLines)
- {
- int len = line[ix].Length + 1;
- if (pos < lstart + len) break;
- lstart += len;
- ix++;
- }
- cachedPosition = pos;
- cachedIxdex = ix;
- cachedLineStart = lstart;
- }
-
- public override string GetString(int begin, int limit)
- {
- if (begin >= maxPos || limit <= begin) return "";
- int endIx, begIx, endLineStart, begLineStart;
- findIndex(begin, out begIx, out begLineStart);
- int begCol = begin - begLineStart;
- findIndex(limit, out endIx, out endLineStart);
- int endCol = limit - endLineStart;
- string s = line[begIx];
- if (begIx == endIx)
- {
- // the usual case, substring all on one line
- return (endCol <= s.Length) ?
- s.Substring(begCol, endCol - begCol)
- : s.Substring(begCol) + "\n";
- }
- // the string spans multiple lines, yuk!
- StringBuilder sb = new StringBuilder();
- if (begCol < s.Length)
- sb.Append(s.Substring(begCol));
- for (; ; )
- {
- sb.Append("\n");
- s = line[++begIx];
- if (begIx >= endIx) break;
- sb.Append(s);
- }
- if (endCol <= s.Length)
- {
- sb.Append(s.Substring(0, endCol));
- }
- else
- {
- sb.Append(s);
- sb.Append("\n");
- }
- return sb.ToString();
- }
-
- public override int Pos
- {
- get { return cPos; }
- set
- {
- cPos = value;
- findIndex(cPos, out cLine, out curLineStart);
- // cLine should be the *next* line after curLine.
- curLine = (cLine < numLines ? line[cLine++] : "");
- curLineEnd = curLineStart + curLine.Length;
- }
- }
-
- public override string ToString() { return "LineBuffer"; }
- }
-
-#if (!NOFILES)
- // ==============================================================
- // ===== class BuildBuff : for unicode text files ========
- // ==============================================================
-
- class BuildBuffer : ScanBuff
- {
- // Double buffer for char stream.
- class BufferElement
- {
- StringBuilder bldr = new StringBuilder();
- StringBuilder next = new StringBuilder();
- int minIx;
- int maxIx;
- int brkIx;
- bool appendToNext;
-
- internal BufferElement() { }
-
- internal int MaxIndex { get { return maxIx; } }
- // internal int MinIndex { get { return minIx; } }
-
- internal char this[int index]
- {
- get
- {
- if (index < minIx || index >= maxIx)
- throw new BufferException("Index was outside data buffer");
- else if (index < brkIx)
- return bldr[index - minIx];
- else
- return next[index - brkIx];
- }
- }
-
- internal void Append(char[] block, int count)
- {
- maxIx += count;
- if (appendToNext)
- this.next.Append(block, 0, count);
- else
- {
- this.bldr.Append(block, 0, count);
- brkIx = maxIx;
- appendToNext = true;
- }
- }
-
- internal string GetString(int start, int limit)
- {
- if (limit <= start)
- return "";
- if (start >= minIx && limit <= maxIx)
- if (limit < brkIx) // String entirely in bldr builder
- return bldr.ToString(start - minIx, limit - start);
- else if (start >= brkIx) // String entirely in next builder
- return next.ToString(start - brkIx, limit - start);
- else // Must do a string-concatenation
- return
- bldr.ToString(start - minIx, brkIx - start) +
- next.ToString(0, limit - brkIx);
- else
- throw new BufferException("String was outside data buffer");
- }
-
- internal void Mark(int limit)
- {
- if (limit > brkIx + 16) // Rotate blocks
- {
- StringBuilder temp = bldr;
- bldr = next;
- next = temp;
- next.Length = 0;
- minIx = brkIx;
- brkIx = maxIx;
- }
- }
- }
-
- BufferElement data = new BufferElement();
-
- int bPos; // Postion index in the StringBuilder
- BlockReader NextBlk; // Delegate that serves char-arrays;
-
- private string EncodingName
- {
- get
- {
- StreamReader rdr = NextBlk.Target as StreamReader;
- return (rdr == null ? "raw-bytes" : rdr.CurrentEncoding.BodyName);
- }
- }
-
- public BuildBuffer(Stream stream)
- {
- FileStream fStrm = (stream as FileStream);
- if (fStrm != null) FileName = fStrm.Name;
- NextBlk = BlockReaderFactory.Raw(stream);
- }
-
-#if (!BYTEMODE)
- public BuildBuffer(Stream stream, int fallbackCodePage)
- {
- FileStream fStrm = (stream as FileStream);
- if (fStrm != null) FileName = fStrm.Name;
- NextBlk = BlockReaderFactory.Get(stream, fallbackCodePage);
- }
-#endif
-
- ///
- /// Marks a conservative lower bound for the buffer,
- /// allowing space to be reclaimed. If an application
- /// needs to call GetString at arbitrary past locations
- /// in the input stream, Mark() is not called.
- ///
- public override void Mark() { data.Mark(bPos - 2); }
-
- public override int Pos
- {
- get { return bPos; }
- set { bPos = value; }
- }
-
-
- ///
- /// Read returns the ordinal number of the next char, or
- /// EOF (-1) for an end of stream. Note that the next
- /// code point may require *two* calls of Read().
- ///
- ///
- public override int Read()
- {
- //
- // Characters at positions
- // [data.offset, data.offset + data.bldr.Length)
- // are available in data.bldr.
- //
- if (bPos < data.MaxIndex)
- {
- // ch0 cannot be EOF
- return (int)data[bPos++];
- }
- else // Read from underlying stream
- {
- // Experimental code, blocks of page size
- char[] chrs = new char[4096];
- int count = NextBlk(chrs, 0, 4096);
- if (count == 0)
- return EndOfFile;
- else
- {
- data.Append(chrs, count);
- return (int)data[bPos++];
- }
- }
- }
-
- public override string GetString(int begin, int limit)
- {
- return data.GetString(begin, limit);
- }
-
- public override string ToString()
- {
- return "StringBuilder buffer, encoding: " + this.EncodingName;
- }
- }
-
- // =============== End ScanBuff-derived classes ==================
-
- internal delegate int BlockReader(char[] block, int index, int number);
-
- // A delegate factory, serving up a delegate that
- // reads a block of characters from the underlying
- // encoded stream, via a StreamReader object.
- //
- internal static class BlockReaderFactory
- {
- public static BlockReader Raw(Stream stream)
- {
- return delegate(char[] block, int index, int number)
- {
- byte[] b = new byte[number];
- int count = stream.Read(b, 0, number);
- int i = 0;
- int j = index;
- for (; i < count; i++, j++)
- block[j] = (char)b[i];
- return count;
- };
- }
-
-#if (!BYTEMODE)
- public static BlockReader Get(Stream stream, int fallbackCodePage)
- {
- Encoding encoding;
- int preamble = Preamble(stream);
-
- if (preamble != 0) // There is a valid BOM here!
- encoding = Encoding.GetEncoding(preamble);
- else if (fallbackCodePage == -1) // Fallback is "raw" bytes
- return Raw(stream);
- else if (fallbackCodePage != -2) // Anything but "guess"
- encoding = Encoding.GetEncoding(fallbackCodePage);
- else // This is the "guess" option
- {
- int guess = new Guesser(stream).GuessCodePage();
- stream.Seek(0, SeekOrigin.Begin);
- if (guess == -1) // ==> this is a 7-bit file
- encoding = Encoding.ASCII;
- else if (guess == 65001)
- encoding = Encoding.UTF8;
- else // ==> use the machine default
- encoding = Encoding.Default;
- }
- StreamReader reader = new StreamReader(stream, encoding);
- return reader.Read;
- }
-
- static int Preamble(Stream stream)
- {
- int b0 = stream.ReadByte();
- int b1 = stream.ReadByte();
-
- if (b0 == 0xfe && b1 == 0xff)
- return 1201; // UTF16BE
- if (b0 == 0xff && b1 == 0xfe)
- return 1200; // UTF16LE
-
- int b2 = stream.ReadByte();
- if (b0 == 0xef && b1 == 0xbb && b2 == 0xbf)
- return 65001; // UTF8
- //
- // There is no unicode preamble, so we
- // return denoter for the machine default.
- //
- stream.Seek(0, SeekOrigin.Begin);
- return 0;
- }
-#endif // !BYTEMODE
- }
-#endif // !NOFILES
- #endregion Buffer classes
-
- // ==============================================================
- // ============ class CodePageHandling =============
- // ==============================================================
-#if (!NOFILES)
- public static class CodePageHandling
- {
- public static int GetCodePage(string option)
- {
- string command = option.ToUpperInvariant();
- if (command.StartsWith("CodePage:", StringComparison.OrdinalIgnoreCase))
- command = command.Substring(9);
- try
- {
- if (command.Equals("RAW"))
- return -1;
- else if (command.Equals("GUESS"))
- return -2;
- else if (command.Equals("DEFAULT"))
- return 0;
- else if (char.IsDigit(command[0]))
- return int.Parse(command, CultureInfo.InvariantCulture);
- else
- {
- Encoding enc = Encoding.GetEncoding(command);
- return enc.CodePage;
- }
- }
- catch (FormatException)
- {
- Console.Error.WriteLine(
- "Invalid format \"{0}\", using machine default", option);
- }
- catch (ArgumentException)
- {
- Console.Error.WriteLine(
- "Unknown code page \"{0}\", using machine default", option);
- }
- return 0;
- }
- }
-#region guesser
-#if (!BYTEMODE)
- // ==============================================================
- // ============ Encoding Guesser =============
- // ==============================================================
-
- ///
- /// This class provides a simple finite state automaton that
- /// scans the file looking for (1) valid UTF-8 byte patterns,
- /// (2) bytes >= 0x80 which are not part of a UTF-8 sequence.
- /// The method then guesses whether it is UTF-8 or maybe some
- /// local machine default encoding. This works well for the
- /// various Latin encodings.
- ///
- internal class Guesser
- {
- ScanBuff buffer;
-
- public int GuessCodePage() { return Scan(); }
-
- const int maxAccept = 10;
- const int initial = 0;
- const int eofNum = 0;
- const int goStart = -1;
- const int INITIAL = 0;
- const int EndToken = 0;
-
- #region user code
- /*
- * Reads the bytes of a file to determine if it is
- * UTF-8 or a single-byte code page file.
- */
- public long utfX;
- public long uppr;
- #endregion user code
-
- int state;
- int currentStart = startState[0];
- int code;
-
- #region ScannerTables
- static int[] startState = new int[] { 11, 0 };
-
- #region CharacterMap
- static sbyte[] map = new sbyte[256] {
-/* '\0' */ 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0,
-/* '\x10' */ 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0,
-/* '\x20' */ 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0,
-/* '0' */ 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0,
-/* '@' */ 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0,
-/* 'P' */ 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0,
-/* '`' */ 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0,
-/* 'p' */ 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0,
-/* '\x80' */ 2, 2, 2, 2, 2, 2, 2, 2, 2, 2, 2, 2, 2, 2, 2, 2,
-/* '\x90' */ 2, 2, 2, 2, 2, 2, 2, 2, 2, 2, 2, 2, 2, 2, 2, 2,
-/* '\xA0' */ 2, 2, 2, 2, 2, 2, 2, 2, 2, 2, 2, 2, 2, 2, 2, 2,
-/* '\xB0' */ 2, 2, 2, 2, 2, 2, 2, 2, 2, 2, 2, 2, 2, 2, 2, 2,
-/* '\xC0' */ 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1,
-/* '\xD0' */ 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1,
-/* '\xE0' */ 3, 3, 3, 3, 3, 3, 3, 3, 3, 3, 3, 3, 3, 3, 3, 3,
-/* '\xF0' */ 4, 4, 4, 4, 4, 4, 4, 4, 5, 5, 5, 5, 5, 5, 5, 5 };
- #endregion
-
- static sbyte[][] nextState = new sbyte[][] {
- new sbyte[] {0, 0, 0, 0, 0, 0},
- new sbyte[] {-1, -1, 10, -1, -1, -1},
- new sbyte[] {-1, -1, -1, -1, -1, -1},
- new sbyte[] {-1, -1, 8, -1, -1, -1},
- new sbyte[] {-1, -1, 5, -1, -1, -1},
- new sbyte[] {-1, -1, 6, -1, -1, -1},
- new sbyte[] {-1, -1, 7, -1, -1, -1},
- null,
- new sbyte[] {-1, -1, 9, -1, -1, -1},
- null,
- null,
- new sbyte[] {-1, 1, 2, 3, 4, 2}
- };
-
-
- [SuppressMessage("Microsoft.Performance", "CA1810:InitializeReferenceTypeStaticFieldsInline")]
- // Reason for suppression: cannot have self-reference in array initializer.
- static Guesser()
- {
- nextState[7] = nextState[2];
- nextState[9] = nextState[2];
- nextState[10] = nextState[2];
- }
-
- int NextState()
- {
- if (code == ScanBuff.EndOfFile)
- return eofNum;
- else
- return nextState[state][map[code]];
- }
- #endregion
-
- public Guesser(System.IO.Stream file) { SetSource(file); }
-
- public void SetSource(System.IO.Stream source)
- {
- this.buffer = new BuildBuffer(source);
- code = buffer.Read();
- }
-
- int Scan()
- {
- for (; ; )
- {
- int next;
- state = currentStart;
- while ((next = NextState()) == goStart)
- code = buffer.Read();
-
- state = next;
- code = buffer.Read();
-
- while ((next = NextState()) > eofNum)
- {
- state = next;
- code = buffer.Read();
- }
- if (state <= maxAccept)
- {
- #region ActionSwitch
-#pragma warning disable 162
- switch (state)
- {
- case eofNum:
- switch (currentStart)
- {
- case 11:
- if (utfX == 0 && uppr == 0) return -1; /* raw ascii */
- else if (uppr * 10 > utfX) return 0; /* default code page */
- else return 65001; /* UTF-8 encoding */
- break;
- }
- return EndToken;
- case 1: // Recognized '{Upper128}', Shortest string "\xC0"
- case 2: // Recognized '{Upper128}', Shortest string "\x80"
- case 3: // Recognized '{Upper128}', Shortest string "\xE0"
- case 4: // Recognized '{Upper128}', Shortest string "\xF0"
- uppr++;
- break;
- case 5: // Recognized '{Utf8pfx4}{Utf8cont}', Shortest string "\xF0\x80"
- uppr += 2;
- break;
- case 6: // Recognized '{Utf8pfx4}{Utf8cont}{2}', Shortest string "\xF0\x80\x80"
- uppr += 3;
- break;
- case 7: // Recognized '{Utf8pfx4}{Utf8cont}{3}', Shortest string "\xF0\x80\x80\x80"
- utfX += 3;
- break;
- case 8: // Recognized '{Utf8pfx3}{Utf8cont}', Shortest string "\xE0\x80"
- uppr += 2;
- break;
- case 9: // Recognized '{Utf8pfx3}{Utf8cont}{2}', Shortest string "\xE0\x80\x80"
- utfX += 2;
- break;
- case 10: // Recognized '{Utf8pfx2}{Utf8cont}', Shortest string "\xC0\x80"
- utfX++;
- break;
- default:
- break;
- }
-#pragma warning restore 162
- #endregion
- }
- }
- }
- } // end class Guesser
-
-#endif // !BYTEMODE
-#endregion
-#endif // !NOFILES
-
-// End of code copied from embedded resource
-}
+// ==============================================================
+//
+// This code automatically produced from an embedded resource.
+// Do not edit this file, or it will become incompatible with
+// the specification from which it was generated.
+//
+// ==============================================================
+
+using System;
+using System.IO;
+using System.Text;
+using System.Collections.Generic;
+using System.Diagnostics.CodeAnalysis;
+using System.Runtime.Serialization;
+using System.Globalization;
+
+namespace QUT.GplexBuffers
+{
+// Code copied from GPLEX embedded resource
+ [Serializable]
+ internal class BufferException : Exception
+ {
+ public BufferException() { }
+ public BufferException(string message) : base(message) { }
+ public BufferException(string message, Exception innerException)
+ : base(message, innerException) { }
+ protected BufferException(SerializationInfo info, StreamingContext context)
+ : base(info, context) { }
+ }
+
+ internal abstract class ScanBuff
+ {
+ private string fileNm;
+
+ public const int EndOfFile = -1;
+ public const int UnicodeReplacementChar = 0xFFFD;
+
+ public bool IsFile { get { return (fileNm != null); } }
+ public string FileName { get { return fileNm; } set { fileNm = value; } }
+
+ public abstract int Pos { get; set; }
+ public abstract int Read();
+ public virtual void Mark() { }
+
+ public abstract string GetString(int begin, int limit);
+
+ public static ScanBuff GetBuffer(string source)
+ {
+ return new StringBuffer(source);
+ }
+
+ public static ScanBuff GetBuffer(IList source)
+ {
+ return new LineBuffer(source);
+ }
+
+#if (!NOFILES)
+ public static ScanBuff GetBuffer(Stream source)
+ {
+ return new BuildBuffer(source);
+ }
+
+#if (!BYTEMODE)
+ public static ScanBuff GetBuffer(Stream source, int fallbackCodePage)
+ {
+ return new BuildBuffer(source, fallbackCodePage);
+ }
+#endif // !BYTEMODE
+#endif // !NOFILES
+ }
+
+ #region Buffer classes
+
+ // ==============================================================
+ // ===== Definitions for various ScanBuff derived classes ====
+ // ==============================================================
+ // =============== String input ================
+ // ==============================================================
+
+ ///
+ /// This class reads characters from a single string as
+ /// required, for example, by Visual Studio language services
+ ///
+ sealed class StringBuffer : ScanBuff
+ {
+ string str; // input buffer
+ int bPos; // current position in buffer
+ int sLen;
+
+ public StringBuffer(string source)
+ {
+ this.str = source;
+ this.sLen = source.Length;
+ this.FileName = null;
+ }
+
+ public override int Read()
+ {
+ if (bPos < sLen) return str[bPos++];
+ else if (bPos == sLen) { bPos++; return '\n'; } // one strike, see new line
+ else { bPos++; return EndOfFile; } // two strikes and you're out!
+ }
+
+ public override string GetString(int begin, int limit)
+ {
+ // "limit" can be greater than sLen with the BABEL
+ // option set. Read returns a "virtual" EOL if
+ // an attempt is made to read past the end of the
+ // string buffer. Without the guard any attempt
+ // to fetch yytext for a token that includes the
+ // EOL will throw an index exception.
+ if (limit > sLen) limit = sLen;
+ if (limit <= begin) return "";
+ else return str.Substring(begin, limit - begin);
+ }
+
+ public override int Pos
+ {
+ get { return bPos; }
+ set { bPos = value; }
+ }
+
+ public override string ToString() { return "StringBuffer"; }
+ }
+
+ // ==============================================================
+ // The LineBuff class contributed by Nigel Horspool,
+ // nigelh@cs.uvic.cs
+ // ==============================================================
+
+ sealed class LineBuffer : ScanBuff
+ {
+ IList line; // list of source lines from a file
+ int numLines; // number of strings in line list
+ string curLine; // current line in that list
+ int cLine; // index of current line in the list
+ int curLen; // length of current line
+ int curLineStart; // position of line start in whole file
+ int curLineEnd; // position of line end in whole file
+ int maxPos; // max position ever visited in whole file
+ int cPos; // ordinal number of code in source
+
+ // Constructed from a list of strings, one per source line.
+ // The lines have had trailing '\n' characters removed.
+ public LineBuffer(IList lineList)
+ {
+ line = lineList;
+ numLines = line.Count;
+ cPos = curLineStart = 0;
+ curLine = (numLines > 0 ? line[0] : "");
+ maxPos = curLineEnd = curLen = curLine.Length;
+ cLine = 1;
+ FileName = null;
+ }
+
+ public override int Read()
+ {
+ if (cPos < curLineEnd)
+ return curLine[cPos++ - curLineStart];
+ if (cPos++ == curLineEnd)
+ return '\n';
+ if (cLine >= numLines)
+ return EndOfFile;
+ curLine = line[cLine];
+ curLen = curLine.Length;
+ curLineStart = curLineEnd + 1;
+ curLineEnd = curLineStart + curLen;
+ if (curLineEnd > maxPos)
+ maxPos = curLineEnd;
+ cLine++;
+ return curLen > 0 ? curLine[0] : '\n';
+ }
+
+ // To speed up searches for the line containing a position
+ private int cachedPosition;
+ private int cachedIxdex;
+ private int cachedLineStart;
+
+ // Given a position pos within the entire source, the results are
+ // ix -- the index of the containing line
+ // lstart -- the position of the first character on that line
+ private void findIndex(int pos, out int ix, out int lstart)
+ {
+ if (pos >= cachedPosition)
+ {
+ ix = cachedIxdex; lstart = cachedLineStart;
+ }
+ else
+ {
+ ix = lstart = 0;
+ }
+ while (ix < numLines)
+ {
+ int len = line[ix].Length + 1;
+ if (pos < lstart + len) break;
+ lstart += len;
+ ix++;
+ }
+ cachedPosition = pos;
+ cachedIxdex = ix;
+ cachedLineStart = lstart;
+ }
+
+ public override string GetString(int begin, int limit)
+ {
+ if (begin >= maxPos || limit <= begin) return "";
+ int endIx, begIx, endLineStart, begLineStart;
+ findIndex(begin, out begIx, out begLineStart);
+ int begCol = begin - begLineStart;
+ findIndex(limit, out endIx, out endLineStart);
+ int endCol = limit - endLineStart;
+ string s = line[begIx];
+ if (begIx == endIx)
+ {
+ // the usual case, substring all on one line
+ return (endCol <= s.Length) ?
+ s.Substring(begCol, endCol - begCol)
+ : s.Substring(begCol) + "\n";
+ }
+ // the string spans multiple lines, yuk!
+ StringBuilder sb = new StringBuilder();
+ if (begCol < s.Length)
+ sb.Append(s.Substring(begCol));
+ for (; ; )
+ {
+ sb.Append("\n");
+ s = line[++begIx];
+ if (begIx >= endIx) break;
+ sb.Append(s);
+ }
+ if (endCol <= s.Length)
+ {
+ sb.Append(s.Substring(0, endCol));
+ }
+ else
+ {
+ sb.Append(s);
+ sb.Append("\n");
+ }
+ return sb.ToString();
+ }
+
+ public override int Pos
+ {
+ get { return cPos; }
+ set
+ {
+ cPos = value;
+ findIndex(cPos, out cLine, out curLineStart);
+ // cLine should be the *next* line after curLine.
+ curLine = (cLine < numLines ? line[cLine++] : "");
+ curLineEnd = curLineStart + curLine.Length;
+ }
+ }
+
+ public override string ToString() { return "LineBuffer"; }
+ }
+
+#if (!NOFILES)
+ // ==============================================================
+ // ===== class BuildBuff : for unicode text files ========
+ // ==============================================================
+
+ class BuildBuffer : ScanBuff
+ {
+ // Double buffer for char stream.
+ class BufferElement
+ {
+ StringBuilder bldr = new StringBuilder();
+ StringBuilder next = new StringBuilder();
+ int minIx;
+ int maxIx;
+ int brkIx;
+ bool appendToNext;
+
+ internal BufferElement() { }
+
+ internal int MaxIndex { get { return maxIx; } }
+ // internal int MinIndex { get { return minIx; } }
+
+ internal char this[int index]
+ {
+ get
+ {
+ if (index < minIx || index >= maxIx)
+ throw new BufferException("Index was outside data buffer");
+ else if (index < brkIx)
+ return bldr[index - minIx];
+ else
+ return next[index - brkIx];
+ }
+ }
+
+ internal void Append(char[] block, int count)
+ {
+ maxIx += count;
+ if (appendToNext)
+ this.next.Append(block, 0, count);
+ else
+ {
+ this.bldr.Append(block, 0, count);
+ brkIx = maxIx;
+ appendToNext = true;
+ }
+ }
+
+ internal string GetString(int start, int limit)
+ {
+ if (limit <= start)
+ return "";
+ if (start >= minIx && limit <= maxIx)
+ if (limit < brkIx) // String entirely in bldr builder
+ return bldr.ToString(start - minIx, limit - start);
+ else if (start >= brkIx) // String entirely in next builder
+ return next.ToString(start - brkIx, limit - start);
+ else // Must do a string-concatenation
+ return
+ bldr.ToString(start - minIx, brkIx - start) +
+ next.ToString(0, limit - brkIx);
+ else
+ throw new BufferException("String was outside data buffer");
+ }
+
+ internal void Mark(int limit)
+ {
+ if (limit > brkIx + 16) // Rotate blocks
+ {
+ StringBuilder temp = bldr;
+ bldr = next;
+ next = temp;
+ next.Length = 0;
+ minIx = brkIx;
+ brkIx = maxIx;
+ }
+ }
+ }
+
+ BufferElement data = new BufferElement();
+
+ int bPos; // Postion index in the StringBuilder
+ BlockReader NextBlk; // Delegate that serves char-arrays;
+
+ private string EncodingName
+ {
+ get
+ {
+ StreamReader rdr = NextBlk.Target as StreamReader;
+ return (rdr == null ? "raw-bytes" : rdr.CurrentEncoding.BodyName);
+ }
+ }
+
+ public BuildBuffer(Stream stream)
+ {
+ FileStream fStrm = (stream as FileStream);
+ if (fStrm != null) FileName = fStrm.Name;
+ NextBlk = BlockReaderFactory.Raw(stream);
+ }
+
+#if (!BYTEMODE)
+ public BuildBuffer(Stream stream, int fallbackCodePage)
+ {
+ FileStream fStrm = (stream as FileStream);
+ if (fStrm != null) FileName = fStrm.Name;
+ NextBlk = BlockReaderFactory.Get(stream, fallbackCodePage);
+ }
+#endif
+
+ ///
+ /// Marks a conservative lower bound for the buffer,
+ /// allowing space to be reclaimed. If an application
+ /// needs to call GetString at arbitrary past locations
+ /// in the input stream, Mark() is not called.
+ ///
+ public override void Mark() { data.Mark(bPos - 2); }
+
+ public override int Pos
+ {
+ get { return bPos; }
+ set { bPos = value; }
+ }
+
+
+ ///
+ /// Read returns the ordinal number of the next char, or
+ /// EOF (-1) for an end of stream. Note that the next
+ /// code point may require *two* calls of Read().
+ ///
+ ///
+ public override int Read()
+ {
+ //
+ // Characters at positions
+ // [data.offset, data.offset + data.bldr.Length)
+ // are available in data.bldr.
+ //
+ if (bPos < data.MaxIndex)
+ {
+ // ch0 cannot be EOF
+ return (int)data[bPos++];
+ }
+ else // Read from underlying stream
+ {
+ // Experimental code, blocks of page size
+ char[] chrs = new char[4096];
+ int count = NextBlk(chrs, 0, 4096);
+ if (count == 0)
+ return EndOfFile;
+ else
+ {
+ data.Append(chrs, count);
+ return (int)data[bPos++];
+ }
+ }
+ }
+
+ public override string GetString(int begin, int limit)
+ {
+ return data.GetString(begin, limit);
+ }
+
+ public override string ToString()
+ {
+ return "StringBuilder buffer, encoding: " + this.EncodingName;
+ }
+ }
+
+ // =============== End ScanBuff-derived classes ==================
+
+ internal delegate int BlockReader(char[] block, int index, int number);
+
+ // A delegate factory, serving up a delegate that
+ // reads a block of characters from the underlying
+ // encoded stream, via a StreamReader object.
+ //
+ internal static class BlockReaderFactory
+ {
+ public static BlockReader Raw(Stream stream)
+ {
+ return delegate(char[] block, int index, int number)
+ {
+ byte[] b = new byte[number];
+ int count = stream.Read(b, 0, number);
+ int i = 0;
+ int j = index;
+ for (; i < count; i++, j++)
+ block[j] = (char)b[i];
+ return count;
+ };
+ }
+
+#if (!BYTEMODE)
+ public static BlockReader Get(Stream stream, int fallbackCodePage)
+ {
+ Encoding encoding;
+ int preamble = Preamble(stream);
+
+ if (preamble != 0) // There is a valid BOM here!
+ encoding = Encoding.GetEncoding(preamble);
+ else if (fallbackCodePage == -1) // Fallback is "raw" bytes
+ return Raw(stream);
+ else if (fallbackCodePage != -2) // Anything but "guess"
+ encoding = Encoding.GetEncoding(fallbackCodePage);
+ else // This is the "guess" option
+ {
+ int guess = new Guesser(stream).GuessCodePage();
+ stream.Seek(0, SeekOrigin.Begin);
+ if (guess == -1) // ==> this is a 7-bit file
+ encoding = Encoding.ASCII;
+ else if (guess == 65001)
+ encoding = Encoding.UTF8;
+ else // ==> use the machine default
+ encoding = Encoding.Default;
+ }
+ StreamReader reader = new StreamReader(stream, encoding);
+ return reader.Read;
+ }
+
+ static int Preamble(Stream stream)
+ {
+ int b0 = stream.ReadByte();
+ int b1 = stream.ReadByte();
+
+ if (b0 == 0xfe && b1 == 0xff)
+ return 1201; // UTF16BE
+ if (b0 == 0xff && b1 == 0xfe)
+ return 1200; // UTF16LE
+
+ int b2 = stream.ReadByte();
+ if (b0 == 0xef && b1 == 0xbb && b2 == 0xbf)
+ return 65001; // UTF8
+ //
+ // There is no unicode preamble, so we
+ // return denoter for the machine default.
+ //
+ stream.Seek(0, SeekOrigin.Begin);
+ return 0;
+ }
+#endif // !BYTEMODE
+ }
+#endif // !NOFILES
+ #endregion Buffer classes
+
+ // ==============================================================
+ // ============ class CodePageHandling =============
+ // ==============================================================
+#if (!NOFILES)
+ public static class CodePageHandling
+ {
+ public static int GetCodePage(string option)
+ {
+ string command = option.ToUpperInvariant();
+ if (command.StartsWith("CodePage:", StringComparison.OrdinalIgnoreCase))
+ command = command.Substring(9);
+ try
+ {
+ if (command.Equals("RAW"))
+ return -1;
+ else if (command.Equals("GUESS"))
+ return -2;
+ else if (command.Equals("DEFAULT"))
+ return 0;
+ else if (char.IsDigit(command[0]))
+ return int.Parse(command, CultureInfo.InvariantCulture);
+ else
+ {
+ Encoding enc = Encoding.GetEncoding(command);
+ return enc.CodePage;
+ }
+ }
+ catch (FormatException)
+ {
+ Console.Error.WriteLine(
+ "Invalid format \"{0}\", using machine default", option);
+ }
+ catch (ArgumentException)
+ {
+ Console.Error.WriteLine(
+ "Unknown code page \"{0}\", using machine default", option);
+ }
+ return 0;
+ }
+ }
+#region guesser
+#if (!BYTEMODE)
+ // ==============================================================
+ // ============ Encoding Guesser =============
+ // ==============================================================
+
+ ///
+ /// This class provides a simple finite state automaton that
+ /// scans the file looking for (1) valid UTF-8 byte patterns,
+ /// (2) bytes >= 0x80 which are not part of a UTF-8 sequence.
+ /// The method then guesses whether it is UTF-8 or maybe some
+ /// local machine default encoding. This works well for the
+ /// various Latin encodings.
+ ///
+ internal class Guesser
+ {
+ ScanBuff buffer;
+
+ public int GuessCodePage() { return Scan(); }
+
+ const int maxAccept = 10;
+ const int initial = 0;
+ const int eofNum = 0;
+ const int goStart = -1;
+ const int INITIAL = 0;
+ const int EndToken = 0;
+
+ #region user code
+ /*
+ * Reads the bytes of a file to determine if it is
+ * UTF-8 or a single-byte code page file.
+ */
+ public long utfX;
+ public long uppr;
+ #endregion user code
+
+ int state;
+ int currentStart = startState[0];
+ int code;
+
+ #region ScannerTables
+ static int[] startState = new int[] { 11, 0 };
+
+ #region CharacterMap
+ static sbyte[] map = new sbyte[256] {
+/* '\0' */ 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0,
+/* '\x10' */ 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0,
+/* '\x20' */ 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0,
+/* '0' */ 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0,
+/* '@' */ 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0,
+/* 'P' */ 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0,
+/* '`' */ 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0,
+/* 'p' */ 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0,
+/* '\x80' */ 2, 2, 2, 2, 2, 2, 2, 2, 2, 2, 2, 2, 2, 2, 2, 2,
+/* '\x90' */ 2, 2, 2, 2, 2, 2, 2, 2, 2, 2, 2, 2, 2, 2, 2, 2,
+/* '\xA0' */ 2, 2, 2, 2, 2, 2, 2, 2, 2, 2, 2, 2, 2, 2, 2, 2,
+/* '\xB0' */ 2, 2, 2, 2, 2, 2, 2, 2, 2, 2, 2, 2, 2, 2, 2, 2,
+/* '\xC0' */ 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1,
+/* '\xD0' */ 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1,
+/* '\xE0' */ 3, 3, 3, 3, 3, 3, 3, 3, 3, 3, 3, 3, 3, 3, 3, 3,
+/* '\xF0' */ 4, 4, 4, 4, 4, 4, 4, 4, 5, 5, 5, 5, 5, 5, 5, 5 };
+ #endregion
+
+ static sbyte[][] nextState = new sbyte[][] {
+ new sbyte[] {0, 0, 0, 0, 0, 0},
+ new sbyte[] {-1, -1, 10, -1, -1, -1},
+ new sbyte[] {-1, -1, -1, -1, -1, -1},
+ new sbyte[] {-1, -1, 8, -1, -1, -1},
+ new sbyte[] {-1, -1, 5, -1, -1, -1},
+ new sbyte[] {-1, -1, 6, -1, -1, -1},
+ new sbyte[] {-1, -1, 7, -1, -1, -1},
+ null,
+ new sbyte[] {-1, -1, 9, -1, -1, -1},
+ null,
+ null,
+ new sbyte[] {-1, 1, 2, 3, 4, 2}
+ };
+
+
+ [SuppressMessage("Microsoft.Performance", "CA1810:InitializeReferenceTypeStaticFieldsInline")]
+ // Reason for suppression: cannot have self-reference in array initializer.
+ static Guesser()
+ {
+ nextState[7] = nextState[2];
+ nextState[9] = nextState[2];
+ nextState[10] = nextState[2];
+ }
+
+ int NextState()
+ {
+ if (code == ScanBuff.EndOfFile)
+ return eofNum;
+ else
+ return nextState[state][map[code]];
+ }
+ #endregion
+
+ public Guesser(System.IO.Stream file) { SetSource(file); }
+
+ public void SetSource(System.IO.Stream source)
+ {
+ this.buffer = new BuildBuffer(source);
+ code = buffer.Read();
+ }
+
+ int Scan()
+ {
+ for (; ; )
+ {
+ int next;
+ state = currentStart;
+ while ((next = NextState()) == goStart)
+ code = buffer.Read();
+
+ state = next;
+ code = buffer.Read();
+
+ while ((next = NextState()) > eofNum)
+ {
+ state = next;
+ code = buffer.Read();
+ }
+ if (state <= maxAccept)
+ {
+ #region ActionSwitch
+#pragma warning disable 162
+ switch (state)
+ {
+ case eofNum:
+ switch (currentStart)
+ {
+ case 11:
+ if (utfX == 0 && uppr == 0) return -1; /* raw ascii */
+ else if (uppr * 10 > utfX) return 0; /* default code page */
+ else return 65001; /* UTF-8 encoding */
+ break;
+ }
+ return EndToken;
+ case 1: // Recognized '{Upper128}', Shortest string "\xC0"
+ case 2: // Recognized '{Upper128}', Shortest string "\x80"
+ case 3: // Recognized '{Upper128}', Shortest string "\xE0"
+ case 4: // Recognized '{Upper128}', Shortest string "\xF0"
+ uppr++;
+ break;
+ case 5: // Recognized '{Utf8pfx4}{Utf8cont}', Shortest string "\xF0\x80"
+ uppr += 2;
+ break;
+ case 6: // Recognized '{Utf8pfx4}{Utf8cont}{2}', Shortest string "\xF0\x80\x80"
+ uppr += 3;
+ break;
+ case 7: // Recognized '{Utf8pfx4}{Utf8cont}{3}', Shortest string "\xF0\x80\x80\x80"
+ utfX += 3;
+ break;
+ case 8: // Recognized '{Utf8pfx3}{Utf8cont}', Shortest string "\xE0\x80"
+ uppr += 2;
+ break;
+ case 9: // Recognized '{Utf8pfx3}{Utf8cont}{2}', Shortest string "\xE0\x80\x80"
+ utfX += 2;
+ break;
+ case 10: // Recognized '{Utf8pfx2}{Utf8cont}', Shortest string "\xC0\x80"
+ utfX++;
+ break;
+ default:
+ break;
+ }
+#pragma warning restore 162
+ #endregion
+ }
+ }
+ }
+ } // end class Guesser
+
+#endif // !BYTEMODE
+#endregion
+#endif // !NOFILES
+
+// End of code copied from embedded resource
+}
diff --git a/src/DataStreamSuite/Bts.ColumnExpressions/Bts.ColumnExpressions.csproj b/src/DataStreamSuite/Bts.ColumnExpressions/Bts.ColumnExpressions.csproj
index 39adf039f..3c12da7d5 100644
--- a/src/DataStreamSuite/Bts.ColumnExpressions/Bts.ColumnExpressions.csproj
+++ b/src/DataStreamSuite/Bts.ColumnExpressions/Bts.ColumnExpressions.csproj
@@ -1,7 +1,7 @@

- net8.0
+ net7.0
enable
enable
diff --git a/src/DataStreamSuite/Bts.ColumnExpressions/ColumnExpression.cs b/src/DataStreamSuite/Bts.ColumnExpressions/ColumnExpression.cs
index ee6a94217..e8c7abced 100644
--- a/src/DataStreamSuite/Bts.ColumnExpressions/ColumnExpression.cs
+++ b/src/DataStreamSuite/Bts.ColumnExpressions/ColumnExpression.cs
@@ -1,15 +1,21 @@
namespace Bts.ColumnExpressions;
-public class ColumnExpression(string name, Type? type = null, object? value = null) {
- public string Name { get; } = name;
+public class ColumnExpression {
+ public ColumnExpression(string name, Type? type = null, object? value = null) {
+ Name = name;
+ Type = type
+ ?? value?.GetType()
+ ?? throw new ArgumentException(
+ "Type must be specified explicitly when value is null", nameof(type));
+ Value = value
+ ?? DBNull.Value;
+ }
- public Type Type { get; } = type
- ?? value?.GetType()
- ?? throw new ArgumentException(
- "Type must be specified explicitly when value is null", nameof(type));
+ public string Name { get; }
- public object Value { get; } = value
- ?? DBNull.Value;
+ public Type Type { get; }
+
+ public object Value { get; }
public static ColumnExpression Create(string name, T value, Type? type = null) {
// Try to convert value to 'type' to verify they are compatible
diff --git a/src/DataStreamSuite/Bts.ColumnExpressions/ColumnExpressionGrammar.y b/src/DataStreamSuite/Bts.ColumnExpressions/ColumnExpressionGrammar.y
index f8c545933..942a62b0e 100644
--- a/src/DataStreamSuite/Bts.ColumnExpressions/ColumnExpressionGrammar.y
+++ b/src/DataStreamSuite/Bts.ColumnExpressions/ColumnExpressionGrammar.y
@@ -29,7 +29,7 @@
%%
ColumnExpressions :
- ColumnExpression { $$ = [$1]; }
+ ColumnExpression { $$ = ImmutableList.CreateRange(new[] { $1 }); }
| ColumnExpressions ColumnExpression { $$ = $1.Add($2); }
;
diff --git a/src/DataStreamSuite/Bts.ColumnSortExpressions/Bts.ColumnSortExpressions.csproj b/src/DataStreamSuite/Bts.ColumnSortExpressions/Bts.ColumnSortExpressions.csproj
index 0702bb4d2..b8d537b2f 100644
--- a/src/DataStreamSuite/Bts.ColumnSortExpressions/Bts.ColumnSortExpressions.csproj
+++ b/src/DataStreamSuite/Bts.ColumnSortExpressions/Bts.ColumnSortExpressions.csproj
@@ -1,7 +1,7 @@

- net8.0
+ net7.0
enable
enable
diff --git a/src/DataStreamSuite/Bts.ColumnSortExpressions/ColumnSortExpression.cs b/src/DataStreamSuite/Bts.ColumnSortExpressions/ColumnSortExpression.cs
index fdb4da2fe..b52f0092c 100644
--- a/src/DataStreamSuite/Bts.ColumnSortExpressions/ColumnSortExpression.cs
+++ b/src/DataStreamSuite/Bts.ColumnSortExpressions/ColumnSortExpression.cs
@@ -1,6 +1,11 @@
namespace Bts.ColumnSortExpressions;
-public class ColumnSortExpression(string name, bool isAscending = true) {
- public string Name { get; } = name;
- public bool IsAscending { get; } = isAscending;
+public class ColumnSortExpression {
+ public ColumnSortExpression(string name, bool isAscending = true) {
+ Name = name;
+ IsAscending = isAscending;
+ }
+
+ public string Name { get; }
+ public bool IsAscending { get; }
}
diff --git a/src/DataStreamSuite/Bts.ColumnSortExpressions/ColumnSortExpressionGrammar.y b/src/DataStreamSuite/Bts.ColumnSortExpressions/ColumnSortExpressionGrammar.y
index 69413d975..e1f38872c 100644
--- a/src/DataStreamSuite/Bts.ColumnSortExpressions/ColumnSortExpressionGrammar.y
+++ b/src/DataStreamSuite/Bts.ColumnSortExpressions/ColumnSortExpressionGrammar.y
@@ -18,7 +18,7 @@
%%
Columns :
- Column { $$ = [$1]; }
+ Column { $$ = ImmutableList.CreateRange(new[] { $1 }); }
| Columns Column { $$ = $1.Add($2); }
;
diff --git a/src/DataStreamSuite/ColumnMapExpressions/GplexBuffers.cs b/src/DataStreamSuite/ColumnMapExpressions/GplexBuffers.cs
index 76dd35443..1bcc08e09 100644
--- a/src/DataStreamSuite/ColumnMapExpressions/GplexBuffers.cs
+++ b/src/DataStreamSuite/ColumnMapExpressions/GplexBuffers.cs
@@ -1,720 +1,720 @@
-// ==============================================================
-//
-// This code automatically produced from an embedded resource.
-// Do not edit this file, or it will become incompatible with
-// the specification from which it was generated.
-//
-// ==============================================================
-
-using System;
-using System.IO;
-using System.Text;
-using System.Collections.Generic;
-using System.Diagnostics.CodeAnalysis;
-using System.Runtime.Serialization;
-using System.Globalization;
-
-namespace QUT.GplexBuffers
-{
-// Code copied from GPLEX embedded resource
- [Serializable]
- internal class BufferException : Exception
- {
- public BufferException() { }
- public BufferException(string message) : base(message) { }
- public BufferException(string message, Exception innerException)
- : base(message, innerException) { }
- protected BufferException(SerializationInfo info, StreamingContext context)
- : base(info, context) { }
- }
-
- internal abstract class ScanBuff
- {
- private string fileNm;
-
- public const int EndOfFile = -1;
- public const int UnicodeReplacementChar = 0xFFFD;
-
- public bool IsFile { get { return (fileNm != null); } }
- public string FileName { get { return fileNm; } set { fileNm = value; } }
-
- public abstract int Pos { get; set; }
- public abstract int Read();
- public virtual void Mark() { }
-
- public abstract string GetString(int begin, int limit);
-
- public static ScanBuff GetBuffer(string source)
- {
- return new StringBuffer(source);
- }
-
- public static ScanBuff GetBuffer(IList source)
- {
- return new LineBuffer(source);
- }
-
-#if (!NOFILES)
- public static ScanBuff GetBuffer(Stream source)
- {
- return new BuildBuffer(source);
- }
-
-#if (!BYTEMODE)
- public static ScanBuff GetBuffer(Stream source, int fallbackCodePage)
- {
- return new BuildBuffer(source, fallbackCodePage);
- }
-#endif // !BYTEMODE
-#endif // !NOFILES
- }
-
- #region Buffer classes
-
- // ==============================================================
- // ===== Definitions for various ScanBuff derived classes ====
- // ==============================================================
- // =============== String input ================
- // ==============================================================
-
- ///
- /// This class reads characters from a single string as
- /// required, for example, by Visual Studio language services
- ///
- sealed class StringBuffer : ScanBuff
- {
- string str; // input buffer
- int bPos; // current position in buffer
- int sLen;
-
- public StringBuffer(string source)
- {
- this.str = source;
- this.sLen = source.Length;
- this.FileName = null;
- }
-
- public override int Read()
- {
- if (bPos < sLen) return str[bPos++];
- else if (bPos == sLen) { bPos++; return '\n'; } // one strike, see new line
- else { bPos++; return EndOfFile; } // two strikes and you're out!
- }
-
- public override string GetString(int begin, int limit)
- {
- // "limit" can be greater than sLen with the BABEL
- // option set. Read returns a "virtual" EOL if
- // an attempt is made to read past the end of the
- // string buffer. Without the guard any attempt
- // to fetch yytext for a token that includes the
- // EOL will throw an index exception.
- if (limit > sLen) limit = sLen;
- if (limit <= begin) return "";
- else return str.Substring(begin, limit - begin);
- }
-
- public override int Pos
- {
- get { return bPos; }
- set { bPos = value; }
- }
-
- public override string ToString() { return "StringBuffer"; }
- }
-
- // ==============================================================
- // The LineBuff class contributed by Nigel Horspool,
- // nigelh@cs.uvic.cs
- // ==============================================================
-
- sealed class LineBuffer : ScanBuff
- {
- IList line; // list of source lines from a file
- int numLines; // number of strings in line list
- string curLine; // current line in that list
- int cLine; // index of current line in the list
- int curLen; // length of current line
- int curLineStart; // position of line start in whole file
- int curLineEnd; // position of line end in whole file
- int maxPos; // max position ever visited in whole file
- int cPos; // ordinal number of code in source
-
- // Constructed from a list of strings, one per source line.
- // The lines have had trailing '\n' characters removed.
- public LineBuffer(IList lineList)
- {
- line = lineList;
- numLines = line.Count;
- cPos = curLineStart = 0;
- curLine = (numLines > 0 ? line[0] : "");
- maxPos = curLineEnd = curLen = curLine.Length;
- cLine = 1;
- FileName = null;
- }
-
- public override int Read()
- {
- if (cPos < curLineEnd)
- return curLine[cPos++ - curLineStart];
- if (cPos++ == curLineEnd)
- return '\n';
- if (cLine >= numLines)
- return EndOfFile;
- curLine = line[cLine];
- curLen = curLine.Length;
- curLineStart = curLineEnd + 1;
- curLineEnd = curLineStart + curLen;
- if (curLineEnd > maxPos)
- maxPos = curLineEnd;
- cLine++;
- return curLen > 0 ? curLine[0] : '\n';
- }
-
- // To speed up searches for the line containing a position
- private int cachedPosition;
- private int cachedIxdex;
- private int cachedLineStart;
-
- // Given a position pos within the entire source, the results are
- // ix -- the index of the containing line
- // lstart -- the position of the first character on that line
- private void findIndex(int pos, out int ix, out int lstart)
- {
- if (pos >= cachedPosition)
- {
- ix = cachedIxdex; lstart = cachedLineStart;
- }
- else
- {
- ix = lstart = 0;
- }
- while (ix < numLines)
- {
- int len = line[ix].Length + 1;
- if (pos < lstart + len) break;
- lstart += len;
- ix++;
- }
- cachedPosition = pos;
- cachedIxdex = ix;
- cachedLineStart = lstart;
- }
-
- public override string GetString(int begin, int limit)
- {
- if (begin >= maxPos || limit <= begin) return "";
- int endIx, begIx, endLineStart, begLineStart;
- findIndex(begin, out begIx, out begLineStart);
- int begCol = begin - begLineStart;
- findIndex(limit, out endIx, out endLineStart);
- int endCol = limit - endLineStart;
- string s = line[begIx];
- if (begIx == endIx)
- {
- // the usual case, substring all on one line
- return (endCol <= s.Length) ?
- s.Substring(begCol, endCol - begCol)
- : s.Substring(begCol) + "\n";
- }
- // the string spans multiple lines, yuk!
- StringBuilder sb = new StringBuilder();
- if (begCol < s.Length)
- sb.Append(s.Substring(begCol));
- for (; ; )
- {
- sb.Append("\n");
- s = line[++begIx];
- if (begIx >= endIx) break;
- sb.Append(s);
- }
- if (endCol <= s.Length)
- {
- sb.Append(s.Substring(0, endCol));
- }
- else
- {
- sb.Append(s);
- sb.Append("\n");
- }
- return sb.ToString();
- }
-
- public override int Pos
- {
- get { return cPos; }
- set
- {
- cPos = value;
- findIndex(cPos, out cLine, out curLineStart);
- // cLine should be the *next* line after curLine.
- curLine = (cLine < numLines ? line[cLine++] : "");
- curLineEnd = curLineStart + curLine.Length;
- }
- }
-
- public override string ToString() { return "LineBuffer"; }
- }
-
-#if (!NOFILES)
- // ==============================================================
- // ===== class BuildBuff : for unicode text files ========
- // ==============================================================
-
- class BuildBuffer : ScanBuff
- {
- // Double buffer for char stream.
- class BufferElement
- {
- StringBuilder bldr = new StringBuilder();
- StringBuilder next = new StringBuilder();
- int minIx;
- int maxIx;
- int brkIx;
- bool appendToNext;
-
- internal BufferElement() { }
-
- internal int MaxIndex { get { return maxIx; } }
- // internal int MinIndex { get { return minIx; } }
-
- internal char this[int index]
- {
- get
- {
- if (index < minIx || index >= maxIx)
- throw new BufferException("Index was outside data buffer");
- else if (index < brkIx)
- return bldr[index - minIx];
- else
- return next[index - brkIx];
- }
- }
-
- internal void Append(char[] block, int count)
- {
- maxIx += count;
- if (appendToNext)
- this.next.Append(block, 0, count);
- else
- {
- this.bldr.Append(block, 0, count);
- brkIx = maxIx;
- appendToNext = true;
- }
- }
-
- internal string GetString(int start, int limit)
- {
- if (limit <= start)
- return "";
- if (start >= minIx && limit <= maxIx)
- if (limit < brkIx) // String entirely in bldr builder
- return bldr.ToString(start - minIx, limit - start);
- else if (start >= brkIx) // String entirely in next builder
- return next.ToString(start - brkIx, limit - start);
- else // Must do a string-concatenation
- return
- bldr.ToString(start - minIx, brkIx - start) +
- next.ToString(0, limit - brkIx);
- else
- throw new BufferException("String was outside data buffer");
- }
-
- internal void Mark(int limit)
- {
- if (limit > brkIx + 16) // Rotate blocks
- {
- StringBuilder temp = bldr;
- bldr = next;
- next = temp;
- next.Length = 0;
- minIx = brkIx;
- brkIx = maxIx;
- }
- }
- }
-
- BufferElement data = new BufferElement();
-
- int bPos; // Postion index in the StringBuilder
- BlockReader NextBlk; // Delegate that serves char-arrays;
-
- private string EncodingName
- {
- get
- {
- StreamReader rdr = NextBlk.Target as StreamReader;
- return (rdr == null ? "raw-bytes" : rdr.CurrentEncoding.BodyName);
- }
- }
-
- public BuildBuffer(Stream stream)
- {
- FileStream fStrm = (stream as FileStream);
- if (fStrm != null) FileName = fStrm.Name;
- NextBlk = BlockReaderFactory.Raw(stream);
- }
-
-#if (!BYTEMODE)
- public BuildBuffer(Stream stream, int fallbackCodePage)
- {
- FileStream fStrm = (stream as FileStream);
- if (fStrm != null) FileName = fStrm.Name;
- NextBlk = BlockReaderFactory.Get(stream, fallbackCodePage);
- }
-#endif
-
- ///
- /// Marks a conservative lower bound for the buffer,
- /// allowing space to be reclaimed. If an application
- /// needs to call GetString at arbitrary past locations
- /// in the input stream, Mark() is not called.
- ///
- public override void Mark() { data.Mark(bPos - 2); }
-
- public override int Pos
- {
- get { return bPos; }
- set { bPos = value; }
- }
-
-
- ///
- /// Read returns the ordinal number of the next char, or
- /// EOF (-1) for an end of stream. Note that the next
- /// code point may require *two* calls of Read().
- ///
- ///
- public override int Read()
- {
- //
- // Characters at positions
- // [data.offset, data.offset + data.bldr.Length)
- // are available in data.bldr.
- //
- if (bPos < data.MaxIndex)
- {
- // ch0 cannot be EOF
- return (int)data[bPos++];
- }
- else // Read from underlying stream
- {
- // Experimental code, blocks of page size
- char[] chrs = new char[4096];
- int count = NextBlk(chrs, 0, 4096);
- if (count == 0)
- return EndOfFile;
- else
- {
- data.Append(chrs, count);
- return (int)data[bPos++];
- }
- }
- }
-
- public override string GetString(int begin, int limit)
- {
- return data.GetString(begin, limit);
- }
-
- public override string ToString()
- {
- return "StringBuilder buffer, encoding: " + this.EncodingName;
- }
- }
-
- // =============== End ScanBuff-derived classes ==================
-
- internal delegate int BlockReader(char[] block, int index, int number);
-
- // A delegate factory, serving up a delegate that
- // reads a block of characters from the underlying
- // encoded stream, via a StreamReader object.
- //
- internal static class BlockReaderFactory
- {
- public static BlockReader Raw(Stream stream)
- {
- return delegate(char[] block, int index, int number)
- {
- byte[] b = new byte[number];
- int count = stream.Read(b, 0, number);
- int i = 0;
- int j = index;
- for (; i < count; i++, j++)
- block[j] = (char)b[i];
- return count;
- };
- }
-
-#if (!BYTEMODE)
- public static BlockReader Get(Stream stream, int fallbackCodePage)
- {
- Encoding encoding;
- int preamble = Preamble(stream);
-
- if (preamble != 0) // There is a valid BOM here!
- encoding = Encoding.GetEncoding(preamble);
- else if (fallbackCodePage == -1) // Fallback is "raw" bytes
- return Raw(stream);
- else if (fallbackCodePage != -2) // Anything but "guess"
- encoding = Encoding.GetEncoding(fallbackCodePage);
- else // This is the "guess" option
- {
- int guess = new Guesser(stream).GuessCodePage();
- stream.Seek(0, SeekOrigin.Begin);
- if (guess == -1) // ==> this is a 7-bit file
- encoding = Encoding.ASCII;
- else if (guess == 65001)
- encoding = Encoding.UTF8;
- else // ==> use the machine default
- encoding = Encoding.Default;
- }
- StreamReader reader = new StreamReader(stream, encoding);
- return reader.Read;
- }
-
- static int Preamble(Stream stream)
- {
- int b0 = stream.ReadByte();
- int b1 = stream.ReadByte();
-
- if (b0 == 0xfe && b1 == 0xff)
- return 1201; // UTF16BE
- if (b0 == 0xff && b1 == 0xfe)
- return 1200; // UTF16LE
-
- int b2 = stream.ReadByte();
- if (b0 == 0xef && b1 == 0xbb && b2 == 0xbf)
- return 65001; // UTF8
- //
- // There is no unicode preamble, so we
- // return denoter for the machine default.
- //
- stream.Seek(0, SeekOrigin.Begin);
- return 0;
- }
-#endif // !BYTEMODE
- }
-#endif // !NOFILES
- #endregion Buffer classes
-
- // ==============================================================
- // ============ class CodePageHandling =============
- // ==============================================================
-#if (!NOFILES)
- public static class CodePageHandling
- {
- public static int GetCodePage(string option)
- {
- string command = option.ToUpperInvariant();
- if (command.StartsWith("CodePage:", StringComparison.OrdinalIgnoreCase))
- command = command.Substring(9);
- try
- {
- if (command.Equals("RAW"))
- return -1;
- else if (command.Equals("GUESS"))
- return -2;
- else if (command.Equals("DEFAULT"))
- return 0;
- else if (char.IsDigit(command[0]))
- return int.Parse(command, CultureInfo.InvariantCulture);
- else
- {
- Encoding enc = Encoding.GetEncoding(command);
- return enc.CodePage;
- }
- }
- catch (FormatException)
- {
- Console.Error.WriteLine(
- "Invalid format \"{0}\", using machine default", option);
- }
- catch (ArgumentException)
- {
- Console.Error.WriteLine(
- "Unknown code page \"{0}\", using machine default", option);
- }
- return 0;
- }
- }
-#region guesser
-#if (!BYTEMODE)
- // ==============================================================
- // ============ Encoding Guesser =============
- // ==============================================================
-
- ///
- /// This class provides a simple finite state automaton that
- /// scans the file looking for (1) valid UTF-8 byte patterns,
- /// (2) bytes >= 0x80 which are not part of a UTF-8 sequence.
- /// The method then guesses whether it is UTF-8 or maybe some
- /// local machine default encoding. This works well for the
- /// various Latin encodings.
- ///
- internal class Guesser
- {
- ScanBuff buffer;
-
- public int GuessCodePage() { return Scan(); }
-
- const int maxAccept = 10;
- const int initial = 0;
- const int eofNum = 0;
- const int goStart = -1;
- const int INITIAL = 0;
- const int EndToken = 0;
-
- #region user code
- /*
- * Reads the bytes of a file to determine if it is
- * UTF-8 or a single-byte code page file.
- */
- public long utfX;
- public long uppr;
- #endregion user code
-
- int state;
- int currentStart = startState[0];
- int code;
-
- #region ScannerTables
- static int[] startState = new int[] { 11, 0 };
-
- #region CharacterMap
- static sbyte[] map = new sbyte[256] {
-/* '\0' */ 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0,
-/* '\x10' */ 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0,
-/* '\x20' */ 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0,
-/* '0' */ 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0,
-/* '@' */ 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0,
-/* 'P' */ 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0,
-/* '`' */ 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0,
-/* 'p' */ 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0,
-/* '\x80' */ 2, 2, 2, 2, 2, 2, 2, 2, 2, 2, 2, 2, 2, 2, 2, 2,
-/* '\x90' */ 2, 2, 2, 2, 2, 2, 2, 2, 2, 2, 2, 2, 2, 2, 2, 2,
-/* '\xA0' */ 2, 2, 2, 2, 2, 2, 2, 2, 2, 2, 2, 2, 2, 2, 2, 2,
-/* '\xB0' */ 2, 2, 2, 2, 2, 2, 2, 2, 2, 2, 2, 2, 2, 2, 2, 2,
-/* '\xC0' */ 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1,
-/* '\xD0' */ 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1,
-/* '\xE0' */ 3, 3, 3, 3, 3, 3, 3, 3, 3, 3, 3, 3, 3, 3, 3, 3,
-/* '\xF0' */ 4, 4, 4, 4, 4, 4, 4, 4, 5, 5, 5, 5, 5, 5, 5, 5 };
- #endregion
-
- static sbyte[][] nextState = new sbyte[][] {
- new sbyte[] {0, 0, 0, 0, 0, 0},
- new sbyte[] {-1, -1, 10, -1, -1, -1},
- new sbyte[] {-1, -1, -1, -1, -1, -1},
- new sbyte[] {-1, -1, 8, -1, -1, -1},
- new sbyte[] {-1, -1, 5, -1, -1, -1},
- new sbyte[] {-1, -1, 6, -1, -1, -1},
- new sbyte[] {-1, -1, 7, -1, -1, -1},
- null,
- new sbyte[] {-1, -1, 9, -1, -1, -1},
- null,
- null,
- new sbyte[] {-1, 1, 2, 3, 4, 2}
- };
-
-
- [SuppressMessage("Microsoft.Performance", "CA1810:InitializeReferenceTypeStaticFieldsInline")]
- // Reason for suppression: cannot have self-reference in array initializer.
- static Guesser()
- {
- nextState[7] = nextState[2];
- nextState[9] = nextState[2];
- nextState[10] = nextState[2];
- }
-
- int NextState()
- {
- if (code == ScanBuff.EndOfFile)
- return eofNum;
- else
- return nextState[state][map[code]];
- }
- #endregion
-
- public Guesser(System.IO.Stream file) { SetSource(file); }
-
- public void SetSource(System.IO.Stream source)
- {
- this.buffer = new BuildBuffer(source);
- code = buffer.Read();
- }
-
- int Scan()
- {
- for (; ; )
- {
- int next;
- state = currentStart;
- while ((next = NextState()) == goStart)
- code = buffer.Read();
-
- state = next;
- code = buffer.Read();
-
- while ((next = NextState()) > eofNum)
- {
- state = next;
- code = buffer.Read();
- }
- if (state <= maxAccept)
- {
- #region ActionSwitch
-#pragma warning disable 162
- switch (state)
- {
- case eofNum:
- switch (currentStart)
- {
- case 11:
- if (utfX == 0 && uppr == 0) return -1; /* raw ascii */
- else if (uppr * 10 > utfX) return 0; /* default code page */
- else return 65001; /* UTF-8 encoding */
- break;
- }
- return EndToken;
- case 1: // Recognized '{Upper128}', Shortest string "\xC0"
- case 2: // Recognized '{Upper128}', Shortest string "\x80"
- case 3: // Recognized '{Upper128}', Shortest string "\xE0"
- case 4: // Recognized '{Upper128}', Shortest string "\xF0"
- uppr++;
- break;
- case 5: // Recognized '{Utf8pfx4}{Utf8cont}', Shortest string "\xF0\x80"
- uppr += 2;
- break;
- case 6: // Recognized '{Utf8pfx4}{Utf8cont}{2}', Shortest string "\xF0\x80\x80"
- uppr += 3;
- break;
- case 7: // Recognized '{Utf8pfx4}{Utf8cont}{3}', Shortest string "\xF0\x80\x80\x80"
- utfX += 3;
- break;
- case 8: // Recognized '{Utf8pfx3}{Utf8cont}', Shortest string "\xE0\x80"
- uppr += 2;
- break;
- case 9: // Recognized '{Utf8pfx3}{Utf8cont}{2}', Shortest string "\xE0\x80\x80"
- utfX += 2;
- break;
- case 10: // Recognized '{Utf8pfx2}{Utf8cont}', Shortest string "\xC0\x80"
- utfX++;
- break;
- default:
- break;
- }
-#pragma warning restore 162
- #endregion
- }
- }
- }
- } // end class Guesser
-
-#endif // !BYTEMODE
-#endregion
-#endif // !NOFILES
-
-// End of code copied from embedded resource
-}
+// ==============================================================
+//
+// This code automatically produced from an embedded resource.
+// Do not edit this file, or it will become incompatible with
+// the specification from which it was generated.
+//
+// ==============================================================
+
+using System;
+using System.IO;
+using System.Text;
+using System.Collections.Generic;
+using System.Diagnostics.CodeAnalysis;
+using System.Runtime.Serialization;
+using System.Globalization;
+
+namespace QUT.GplexBuffers
+{
+// Code copied from GPLEX embedded resource
+ [Serializable]
+ internal class BufferException : Exception
+ {
+ public BufferException() { }
+ public BufferException(string message) : base(message) { }
+ public BufferException(string message, Exception innerException)
+ : base(message, innerException) { }
+ protected BufferException(SerializationInfo info, StreamingContext context)
+ : base(info, context) { }
+ }
+
+ internal abstract class ScanBuff
+ {
+ private string fileNm;
+
+ public const int EndOfFile = -1;
+ public const int UnicodeReplacementChar = 0xFFFD;
+
+ public bool IsFile { get { return (fileNm != null); } }
+ public string FileName { get { return fileNm; } set { fileNm = value; } }
+
+ public abstract int Pos { get; set; }
+ public abstract int Read();
+ public virtual void Mark() { }
+
+ public abstract string GetString(int begin, int limit);
+
+ public static ScanBuff GetBuffer(string source)
+ {
+ return new StringBuffer(source);
+ }
+
+ public static ScanBuff GetBuffer(IList source)
+ {
+ return new LineBuffer(source);
+ }
+
+#if (!NOFILES)
+ public static ScanBuff GetBuffer(Stream source)
+ {
+ return new BuildBuffer(source);
+ }
+
+#if (!BYTEMODE)
+ public static ScanBuff GetBuffer(Stream source, int fallbackCodePage)
+ {
+ return new BuildBuffer(source, fallbackCodePage);
+ }
+#endif // !BYTEMODE
+#endif // !NOFILES
+ }
+
+ #region Buffer classes
+
+ // ==============================================================
+ // ===== Definitions for various ScanBuff derived classes ====
+ // ==============================================================
+ // =============== String input ================
+ // ==============================================================
+
+ ///
+ /// This class reads characters from a single string as
+ /// required, for example, by Visual Studio language services
+ ///
+ sealed class StringBuffer : ScanBuff
+ {
+ string str; // input buffer
+ int bPos; // current position in buffer
+ int sLen;
+
+ public StringBuffer(string source)
+ {
+ this.str = source;
+ this.sLen = source.Length;
+ this.FileName = null;
+ }
+
+ public override int Read()
+ {
+ if (bPos < sLen) return str[bPos++];
+ else if (bPos == sLen) { bPos++; return '\n'; } // one strike, see new line
+ else { bPos++; return EndOfFile; } // two strikes and you're out!
+ }
+
+ public override string GetString(int begin, int limit)
+ {
+ // "limit" can be greater than sLen with the BABEL
+ // option set. Read returns a "virtual" EOL if
+ // an attempt is made to read past the end of the
+ // string buffer. Without the guard any attempt
+ // to fetch yytext for a token that includes the
+ // EOL will throw an index exception.
+ if (limit > sLen) limit = sLen;
+ if (limit <= begin) return "";
+ else return str.Substring(begin, limit - begin);
+ }
+
+ public override int Pos
+ {
+ get { return bPos; }
+ set { bPos = value; }
+ }
+
+ public override string ToString() { return "StringBuffer"; }
+ }
+
+ // ==============================================================
+ // The LineBuff class contributed by Nigel Horspool,
+ // nigelh@cs.uvic.cs
+ // ==============================================================
+
+ sealed class LineBuffer : ScanBuff
+ {
+ IList line; // list of source lines from a file
+ int numLines; // number of strings in line list
+ string curLine; // current line in that list
+ int cLine; // index of current line in the list
+ int curLen; // length of current line
+ int curLineStart; // position of line start in whole file
+ int curLineEnd; // position of line end in whole file
+ int maxPos; // max position ever visited in whole file
+ int cPos; // ordinal number of code in source
+
+ // Constructed from a list of strings, one per source line.
+ // The lines have had trailing '\n' characters removed.
+ public LineBuffer(IList lineList)
+ {
+ line = lineList;
+ numLines = line.Count;
+ cPos = curLineStart = 0;
+ curLine = (numLines > 0 ? line[0] : "");
+ maxPos = curLineEnd = curLen = curLine.Length;
+ cLine = 1;
+ FileName = null;
+ }
+
+ public override int Read()
+ {
+ if (cPos < curLineEnd)
+ return curLine[cPos++ - curLineStart];
+ if (cPos++ == curLineEnd)
+ return '\n';
+ if (cLine >= numLines)
+ return EndOfFile;
+ curLine = line[cLine];
+ curLen = curLine.Length;
+ curLineStart = curLineEnd + 1;
+ curLineEnd = curLineStart + curLen;
+ if (curLineEnd > maxPos)
+ maxPos = curLineEnd;
+ cLine++;
+ return curLen > 0 ? curLine[0] : '\n';
+ }
+
+ // To speed up searches for the line containing a position
+ private int cachedPosition;
+ private int cachedIxdex;
+ private int cachedLineStart;
+
+ // Given a position pos within the entire source, the results are
+ // ix -- the index of the containing line
+ // lstart -- the position of the first character on that line
+ private void findIndex(int pos, out int ix, out int lstart)
+ {
+ if (pos >= cachedPosition)
+ {
+ ix = cachedIxdex; lstart = cachedLineStart;
+ }
+ else
+ {
+ ix = lstart = 0;
+ }
+ while (ix < numLines)
+ {
+ int len = line[ix].Length + 1;
+ if (pos < lstart + len) break;
+ lstart += len;
+ ix++;
+ }
+ cachedPosition = pos;
+ cachedIxdex = ix;
+ cachedLineStart = lstart;
+ }
+
+ public override string GetString(int begin, int limit)
+ {
+ if (begin >= maxPos || limit <= begin) return "";
+ int endIx, begIx, endLineStart, begLineStart;
+ findIndex(begin, out begIx, out begLineStart);
+ int begCol = begin - begLineStart;
+ findIndex(limit, out endIx, out endLineStart);
+ int endCol = limit - endLineStart;
+ string s = line[begIx];
+ if (begIx == endIx)
+ {
+ // the usual case, substring all on one line
+ return (endCol <= s.Length) ?
+ s.Substring(begCol, endCol - begCol)
+ : s.Substring(begCol) + "\n";
+ }
+ // the string spans multiple lines, yuk!
+ StringBuilder sb = new StringBuilder();
+ if (begCol < s.Length)
+ sb.Append(s.Substring(begCol));
+ for (; ; )
+ {
+ sb.Append("\n");
+ s = line[++begIx];
+ if (begIx >= endIx) break;
+ sb.Append(s);
+ }
+ if (endCol <= s.Length)
+ {
+ sb.Append(s.Substring(0, endCol));
+ }
+ else
+ {
+ sb.Append(s);
+ sb.Append("\n");
+ }
+ return sb.ToString();
+ }
+
+ public override int Pos
+ {
+ get { return cPos; }
+ set
+ {
+ cPos = value;
+ findIndex(cPos, out cLine, out curLineStart);
+ // cLine should be the *next* line after curLine.
+ curLine = (cLine < numLines ? line[cLine++] : "");
+ curLineEnd = curLineStart + curLine.Length;
+ }
+ }
+
+ public override string ToString() { return "LineBuffer"; }
+ }
+
+#if (!NOFILES)
+ // ==============================================================
+ // ===== class BuildBuff : for unicode text files ========
+ // ==============================================================
+
+ class BuildBuffer : ScanBuff
+ {
+ // Double buffer for char stream.
+ class BufferElement
+ {
+ StringBuilder bldr = new StringBuilder();
+ StringBuilder next = new StringBuilder();
+ int minIx;
+ int maxIx;
+ int brkIx;
+ bool appendToNext;
+
+ internal BufferElement() { }
+
+ internal int MaxIndex { get { return maxIx; } }
+ // internal int MinIndex { get { return minIx; } }
+
+ internal char this[int index]
+ {
+ get
+ {
+ if (index < minIx || index >= maxIx)
+ throw new BufferException("Index was outside data buffer");
+ else if (index < brkIx)
+ return bldr[index - minIx];
+ else
+ return next[index - brkIx];
+ }
+ }
+
+ internal void Append(char[] block, int count)
+ {
+ maxIx += count;
+ if (appendToNext)
+ this.next.Append(block, 0, count);
+ else
+ {
+ this.bldr.Append(block, 0, count);
+ brkIx = maxIx;
+ appendToNext = true;
+ }
+ }
+
+ internal string GetString(int start, int limit)
+ {
+ if (limit <= start)
+ return "";
+ if (start >= minIx && limit <= maxIx)
+ if (limit < brkIx) // String entirely in bldr builder
+ return bldr.ToString(start - minIx, limit - start);
+ else if (start >= brkIx) // String entirely in next builder
+ return next.ToString(start - brkIx, limit - start);
+ else // Must do a string-concatenation
+ return
+ bldr.ToString(start - minIx, brkIx - start) +
+ next.ToString(0, limit - brkIx);
+ else
+ throw new BufferException("String was outside data buffer");
+ }
+
+ internal void Mark(int limit)
+ {
+ if (limit > brkIx + 16) // Rotate blocks
+ {
+ StringBuilder temp = bldr;
+ bldr = next;
+ next = temp;
+ next.Length = 0;
+ minIx = brkIx;
+ brkIx = maxIx;
+ }
+ }
+ }
+
+ BufferElement data = new BufferElement();
+
+ int bPos; // Postion index in the StringBuilder
+ BlockReader NextBlk; // Delegate that serves char-arrays;
+
+ private string EncodingName
+ {
+ get
+ {
+ StreamReader rdr = NextBlk.Target as StreamReader;
+ return (rdr == null ? "raw-bytes" : rdr.CurrentEncoding.BodyName);
+ }
+ }
+
+ public BuildBuffer(Stream stream)
+ {
+ FileStream fStrm = (stream as FileStream);
+ if (fStrm != null) FileName = fStrm.Name;
+ NextBlk = BlockReaderFactory.Raw(stream);
+ }
+
+#if (!BYTEMODE)
+ public BuildBuffer(Stream stream, int fallbackCodePage)
+ {
+ FileStream fStrm = (stream as FileStream);
+ if (fStrm != null) FileName = fStrm.Name;
+ NextBlk = BlockReaderFactory.Get(stream, fallbackCodePage);
+ }
+#endif
+
+ ///
+ /// Marks a conservative lower bound for the buffer,
+ /// allowing space to be reclaimed. If an application
+ /// needs to call GetString at arbitrary past locations
+ /// in the input stream, Mark() is not called.
+ ///
+ public override void Mark() { data.Mark(bPos - 2); }
+
+ public override int Pos
+ {
+ get { return bPos; }
+ set { bPos = value; }
+ }
+
+
+ ///
+ /// Read returns the ordinal number of the next char, or
+ /// EOF (-1) for an end of stream. Note that the next
+ /// code point may require *two* calls of Read().
+ ///
+ ///
+ public override int Read()
+ {
+ //
+ // Characters at positions
+ // [data.offset, data.offset + data.bldr.Length)
+ // are available in data.bldr.
+ //
+ if (bPos < data.MaxIndex)
+ {
+ // ch0 cannot be EOF
+ return (int)data[bPos++];
+ }
+ else // Read from underlying stream
+ {
+ // Experimental code, blocks of page size
+ char[] chrs = new char[4096];
+ int count = NextBlk(chrs, 0, 4096);
+ if (count == 0)
+ return EndOfFile;
+ else
+ {
+ data.Append(chrs, count);
+ return (int)data[bPos++];
+ }
+ }
+ }
+
+ public override string GetString(int begin, int limit)
+ {
+ return data.GetString(begin, limit);
+ }
+
+ public override string ToString()
+ {
+ return "StringBuilder buffer, encoding: " + this.EncodingName;
+ }
+ }
+
+ // =============== End ScanBuff-derived classes ==================
+
+ internal delegate int BlockReader(char[] block, int index, int number);
+
+ // A delegate factory, serving up a delegate that
+ // reads a block of characters from the underlying
+ // encoded stream, via a StreamReader object.
+ //
+ internal static class BlockReaderFactory
+ {
+ public static BlockReader Raw(Stream stream)
+ {
+ return delegate(char[] block, int index, int number)
+ {
+ byte[] b = new byte[number];
+ int count = stream.Read(b, 0, number);
+ int i = 0;
+ int j = index;
+ for (; i < count; i++, j++)
+ block[j] = (char)b[i];
+ return count;
+ };
+ }
+
+#if (!BYTEMODE)
+ public static BlockReader Get(Stream stream, int fallbackCodePage)
+ {
+ Encoding encoding;
+ int preamble = Preamble(stream);
+
+ if (preamble != 0) // There is a valid BOM here!
+ encoding = Encoding.GetEncoding(preamble);
+ else if (fallbackCodePage == -1) // Fallback is "raw" bytes
+ return Raw(stream);
+ else if (fallbackCodePage != -2) // Anything but "guess"
+ encoding = Encoding.GetEncoding(fallbackCodePage);
+ else // This is the "guess" option
+ {
+ int guess = new Guesser(stream).GuessCodePage();
+ stream.Seek(0, SeekOrigin.Begin);
+ if (guess == -1) // ==> this is a 7-bit file
+ encoding = Encoding.ASCII;
+ else if (guess == 65001)
+ encoding = Encoding.UTF8;
+ else // ==> use the machine default
+ encoding = Encoding.Default;
+ }
+ StreamReader reader = new StreamReader(stream, encoding);
+ return reader.Read;
+ }
+
+ static int Preamble(Stream stream)
+ {
+ int b0 = stream.ReadByte();
+ int b1 = stream.ReadByte();
+
+ if (b0 == 0xfe && b1 == 0xff)
+ return 1201; // UTF16BE
+ if (b0 == 0xff && b1 == 0xfe)
+ return 1200; // UTF16LE
+
+ int b2 = stream.ReadByte();
+ if (b0 == 0xef && b1 == 0xbb && b2 == 0xbf)
+ return 65001; // UTF8
+ //
+ // There is no unicode preamble, so we
+ // return denoter for the machine default.
+ //
+ stream.Seek(0, SeekOrigin.Begin);
+ return 0;
+ }
+#endif // !BYTEMODE
+ }
+#endif // !NOFILES
+ #endregion Buffer classes
+
+ // ==============================================================
+ // ============ class CodePageHandling =============
+ // ==============================================================
+#if (!NOFILES)
+ public static class CodePageHandling
+ {
+ public static int GetCodePage(string option)
+ {
+ string command = option.ToUpperInvariant();
+ if (command.StartsWith("CodePage:", StringComparison.OrdinalIgnoreCase))
+ command = command.Substring(9);
+ try
+ {
+ if (command.Equals("RAW"))
+ return -1;
+ else if (command.Equals("GUESS"))
+ return -2;
+ else if (command.Equals("DEFAULT"))
+ return 0;
+ else if (char.IsDigit(command[0]))
+ return int.Parse(command, CultureInfo.InvariantCulture);
+ else
+ {
+ Encoding enc = Encoding.GetEncoding(command);
+ return enc.CodePage;
+ }
+ }
+ catch (FormatException)
+ {
+ Console.Error.WriteLine(
+ "Invalid format \"{0}\", using machine default", option);
+ }
+ catch (ArgumentException)
+ {
+ Console.Error.WriteLine(
+ "Unknown code page \"{0}\", using machine default", option);
+ }
+ return 0;
+ }
+ }
+#region guesser
+#if (!BYTEMODE)
+ // ==============================================================
+ // ============ Encoding Guesser =============
+ // ==============================================================
+
+ ///
+ /// This class provides a simple finite state automaton that
+ /// scans the file looking for (1) valid UTF-8 byte patterns,
+ /// (2) bytes >= 0x80 which are not part of a UTF-8 sequence.
+ /// The method then guesses whether it is UTF-8 or maybe some
+ /// local machine default encoding. This works well for the
+ /// various Latin encodings.
+ ///
+ internal class Guesser
+ {
+ ScanBuff buffer;
+
+ public int GuessCodePage() { return Scan(); }
+
+ const int maxAccept = 10;
+ const int initial = 0;
+ const int eofNum = 0;
+ const int goStart = -1;
+ const int INITIAL = 0;
+ const int EndToken = 0;
+
+ #region user code
+ /*
+ * Reads the bytes of a file to determine if it is
+ * UTF-8 or a single-byte code page file.
+ */
+ public long utfX;
+ public long uppr;
+ #endregion user code
+
+ int state;
+ int currentStart = startState[0];
+ int code;
+
+ #region ScannerTables
+ static int[] startState = new int[] { 11, 0 };
+
+ #region CharacterMap
+ static sbyte[] map = new sbyte[256] {
+/* '\0' */ 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0,
+/* '\x10' */ 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0,
+/* '\x20' */ 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0,
+/* '0' */ 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0,
+/* '@' */ 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0,
+/* 'P' */ 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0,
+/* '`' */ 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0,
+/* 'p' */ 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0,
+/* '\x80' */ 2, 2, 2, 2, 2, 2, 2, 2, 2, 2, 2, 2, 2, 2, 2, 2,
+/* '\x90' */ 2, 2, 2, 2, 2, 2, 2, 2, 2, 2, 2, 2, 2, 2, 2, 2,
+/* '\xA0' */ 2, 2, 2, 2, 2, 2, 2, 2, 2, 2, 2, 2, 2, 2, 2, 2,
+/* '\xB0' */ 2, 2, 2, 2, 2, 2, 2, 2, 2, 2, 2, 2, 2, 2, 2, 2,
+/* '\xC0' */ 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1,
+/* '\xD0' */ 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1,
+/* '\xE0' */ 3, 3, 3, 3, 3, 3, 3, 3, 3, 3, 3, 3, 3, 3, 3, 3,
+/* '\xF0' */ 4, 4, 4, 4, 4, 4, 4, 4, 5, 5, 5, 5, 5, 5, 5, 5 };
+ #endregion
+
+ static sbyte[][] nextState = new sbyte[][] {
+ new sbyte[] {0, 0, 0, 0, 0, 0},
+ new sbyte[] {-1, -1, 10, -1, -1, -1},
+ new sbyte[] {-1, -1, -1, -1, -1, -1},
+ new sbyte[] {-1, -1, 8, -1, -1, -1},
+ new sbyte[] {-1, -1, 5, -1, -1, -1},
+ new sbyte[] {-1, -1, 6, -1, -1, -1},
+ new sbyte[] {-1, -1, 7, -1, -1, -1},
+ null,
+ new sbyte[] {-1, -1, 9, -1, -1, -1},
+ null,
+ null,
+ new sbyte[] {-1, 1, 2, 3, 4, 2}
+ };
+
+
+ [SuppressMessage("Microsoft.Performance", "CA1810:InitializeReferenceTypeStaticFieldsInline")]
+ // Reason for suppression: cannot have self-reference in array initializer.
+ static Guesser()
+ {
+ nextState[7] = nextState[2];
+ nextState[9] = nextState[2];
+ nextState[10] = nextState[2];
+ }
+
+ int NextState()
+ {
+ if (code == ScanBuff.EndOfFile)
+ return eofNum;
+ else
+ return nextState[state][map[code]];
+ }
+ #endregion
+
+ public Guesser(System.IO.Stream file) { SetSource(file); }
+
+ public void SetSource(System.IO.Stream source)
+ {
+ this.buffer = new BuildBuffer(source);
+ code = buffer.Read();
+ }
+
+ int Scan()
+ {
+ for (; ; )
+ {
+ int next;
+ state = currentStart;
+ while ((next = NextState()) == goStart)
+ code = buffer.Read();
+
+ state = next;
+ code = buffer.Read();
+
+ while ((next = NextState()) > eofNum)
+ {
+ state = next;
+ code = buffer.Read();
+ }
+ if (state <= maxAccept)
+ {
+ #region ActionSwitch
+#pragma warning disable 162
+ switch (state)
+ {
+ case eofNum:
+ switch (currentStart)
+ {
+ case 11:
+ if (utfX == 0 && uppr == 0) return -1; /* raw ascii */
+ else if (uppr * 10 > utfX) return 0; /* default code page */
+ else return 65001; /* UTF-8 encoding */
+ break;
+ }
+ return EndToken;
+ case 1: // Recognized '{Upper128}', Shortest string "\xC0"
+ case 2: // Recognized '{Upper128}', Shortest string "\x80"
+ case 3: // Recognized '{Upper128}', Shortest string "\xE0"
+ case 4: // Recognized '{Upper128}', Shortest string "\xF0"
+ uppr++;
+ break;
+ case 5: // Recognized '{Utf8pfx4}{Utf8cont}', Shortest string "\xF0\x80"
+ uppr += 2;
+ break;
+ case 6: // Recognized '{Utf8pfx4}{Utf8cont}{2}', Shortest string "\xF0\x80\x80"
+ uppr += 3;
+ break;
+ case 7: // Recognized '{Utf8pfx4}{Utf8cont}{3}', Shortest string "\xF0\x80\x80\x80"
+ utfX += 3;
+ break;
+ case 8: // Recognized '{Utf8pfx3}{Utf8cont}', Shortest string "\xE0\x80"
+ uppr += 2;
+ break;
+ case 9: // Recognized '{Utf8pfx3}{Utf8cont}{2}', Shortest string "\xE0\x80\x80"
+ utfX += 2;
+ break;
+ case 10: // Recognized '{Utf8pfx2}{Utf8cont}', Shortest string "\xC0\x80"
+ utfX++;
+ break;
+ default:
+ break;
+ }
+#pragma warning restore 162
+ #endregion
+ }
+ }
+ }
+ } // end class Guesser
+
+#endif // !BYTEMODE
+#endregion
+#endif // !NOFILES
+
+// End of code copied from embedded resource
+}
diff --git a/src/DataStreamSuite/DataStream/Misc/PandasStream.cs b/src/DataStreamSuite/DataStream/Misc/PandasStream.cs
new file mode 100644
index 000000000..a332a787a
--- /dev/null
+++ b/src/DataStreamSuite/DataStream/Misc/PandasStream.cs
@@ -0,0 +1,62 @@
+// using System;
+// using System.Diagnostics;
+
+// using BCG;
+// using BCG.Utilities.General;
+
+
+// namespace bts {
+
+// public class PandasStream : ViewStream, IWorker {
+// //C:\Users\Owner\AppData\Local\Microsoft\WindowsApps\python.exe
+
+// public PandasStream(ViewStream vs, ViewStreamDatabase vsd, string pandas_code) {
+
+// }
+
+// public PandasStream(ViewStream vs, string pandas_code) :
+// this(vs, ViewStreamDatabase.instance, pandas_code) {
+// }
+
+
+// protected override void myOnFirstSubscriber() {
+
+// // string python_code = @"
+// // import winsound
+// // frequency = 2500 # Set frequency (in Hz)
+// // duration = 1000 # Set duration (in milliseconds)
+// // winsound.Beep(frequency, duration)
+// // ";
+// string python_code = @"
+// file_path = \"C:\\Temp\\test2.txt\\" # r prefix for raw string to avoid escape characters
+// with open(file_path, \"w\") as f:
+// f.write(\"Hello, world!\n\")
+// ";
+
+// execute_python_code(python_code);
+// }
+
+// static void execute_python_code(string python_code) {
+// ProcessStartInfo start = new ProcessStartInfo();
+// start.FileName = "python"; // Python executable
+// start.Arguments = "-c \"" + python_code.Replace("\"", "\\\"") + "\""; // Pass code as argument
+// start.UseShellExecute = false;
+// start.RedirectStandardOutput = true;
+
+// using (Process process = Process.Start(start)) {
+// using (System.IO.StreamReader reader = process.StandardOutput) {
+// string result = reader.ReadToEnd();
+// Console.Write(result);
+// }
+// }
+// }
+
+// protected override void myOnLastSubscriber() {
+// }
+
+// public void doWork() {
+
+// }
+
+// }
+// }
\ No newline at end of file
diff --git a/src/DataStreamSuite/DataStream/Misc/test.py b/src/DataStreamSuite/DataStream/Misc/test.py
new file mode 100644
index 000000000..01ae3d5f3
--- /dev/null
+++ b/src/DataStreamSuite/DataStream/Misc/test.py
@@ -0,0 +1,3 @@
+file_path = r"C:\Temp\test2.txt" # r prefix for raw string to avoid escape characters
+with open(file_path, "w") as f:
+ f.write("Hello, world!\n")
\ No newline at end of file
diff --git a/src/DataStreamSuite/DataStream/Primitives/AddColumns/AddColumnsStream.cs b/src/DataStreamSuite/DataStream/Primitives/AddColumns/AddColumnsStream.cs
index 0c4cec642..3c79de595 100644
--- a/src/DataStreamSuite/DataStream/Primitives/AddColumns/AddColumnsStream.cs
+++ b/src/DataStreamSuite/DataStream/Primitives/AddColumns/AddColumnsStream.cs
@@ -1,4 +1,6 @@
namespace BCG;
-public class AddColumnsStream(ViewStream vs, string columnDef)
- : MapStream(vs, new ColumnAdder(columnDef)) { }
+public class AddColumnsStream : MapStream {
+ public AddColumnsStream(ViewStream vs, string columnDef)
+ : base(vs, new ColumnAdder(columnDef)) { }
+}
diff --git a/src/DataStreamSuite/DataStream/Primitives/AddColumns/ColumnAdder.cs b/src/DataStreamSuite/DataStream/Primitives/AddColumns/ColumnAdder.cs
index 35ee9934a..75346aac8 100644
--- a/src/DataStreamSuite/DataStream/Primitives/AddColumns/ColumnAdder.cs
+++ b/src/DataStreamSuite/DataStream/Primitives/AddColumns/ColumnAdder.cs
@@ -13,14 +13,16 @@ namespace BCG;
///
/// String represting the columns to add to the output table
///
-public class ColumnAdder(string columnsExpression) : IStreamMapper {
+public class ColumnAdder : IStreamMapper {
// Store tuples of (column, index) to preserve order
- protected readonly IDictionary myNewColumns =
- (ColumnExpressionInterpreter.Evaluate(columnsExpression) ??
+ protected readonly IDictionary myNewColumns;
+ protected IBlueTableSchema? mySchema;
+
+ public ColumnAdder(string columnsExpression) =>
+ myNewColumns = (ColumnExpressionInterpreter.Evaluate(columnsExpression) ??
throw new Exception("Invalid columnn expression: " + columnsExpression))
.Select((c, i) => (c, i))
.ToDictionary(c => c.c.Name, c => (c.c, c.i));
- protected IBlueTableSchema? mySchema;
[MemberNotNull(nameof(mySchema))]
public virtual IBlueRow map(IBlueRow row) {
diff --git a/src/DataStreamSuite/DataStream/Primitives/GroupBy/MaxStream.cs b/src/DataStreamSuite/DataStream/Primitives/GroupBy/MaxStream.cs
index 9f3b44fc6..8f55cab92 100644
--- a/src/DataStreamSuite/DataStream/Primitives/GroupBy/MaxStream.cs
+++ b/src/DataStreamSuite/DataStream/Primitives/GroupBy/MaxStream.cs
@@ -9,7 +9,7 @@ namespace BCG {
new MaxGroupByEvaluator(outputCol, inputCols)));
}
- // public MaxStream(ViewStream stream, string outputCol)
- // : this(new AddColumnsStream(stream, "fakeCol=0"), outputCol, new string[] { "fakeCol" }) { }
+ public MaxStream(ViewStream stream, string outputCol)
+ : this(new AddColumnsStream(stream, "fakeCol=0"), outputCol, new string[] { "fakeCol" }) { }
}
}
diff --git a/src/DataStreamSuite/DataStream/Primitives/GroupBy/MinStream.cs b/src/DataStreamSuite/DataStream/Primitives/GroupBy/MinStream.cs
index bb3d47495..02eb521af 100644
--- a/src/DataStreamSuite/DataStream/Primitives/GroupBy/MinStream.cs
+++ b/src/DataStreamSuite/DataStream/Primitives/GroupBy/MinStream.cs
@@ -7,7 +7,7 @@ namespace BCG {
new MinGroupByEvaluator(outputCol, inputCol)));
}
- // public MinStream(ViewStream stream, string outputCol)
- // : this(new AddColumnsStream(stream, "fakeCol=0"), outputCol, new string[] { "fakeCol" }) { }
+ public MinStream(ViewStream stream, string outputCol)
+ : this(new AddColumnsStream(stream, "fakeCol=0"), outputCol, new string[] { "fakeCol" }) { }
}
}
diff --git a/src/DataStreamSuite/DataStream/Primitives/GroupBy/SumStream.cs b/src/DataStreamSuite/DataStream/Primitives/GroupBy/SumStream.cs
index dced8d6f5..8ef26e25a 100644
--- a/src/DataStreamSuite/DataStream/Primitives/GroupBy/SumStream.cs
+++ b/src/DataStreamSuite/DataStream/Primitives/GroupBy/SumStream.cs
@@ -31,12 +31,12 @@ namespace BCG {
new MultiColumnSumGroupByEvaluator(spaceSeparatedOutputCols, spaceSeparatedGroupByCols)));
}
- // public SumStream(ViewStream stream, string[] output_columns) {
- // ViewStream rs = new AddColumnsStream(stream, "fake_col=0");
- // rs = new SumStream(rs, output_columns, new string[] { "fake_col" }, false);
- // rs = new ProjectStream(rs, "!fake_col");
- // mySetRealSource(rs);
- // }
+ public SumStream(ViewStream stream, string[] output_columns) {
+ ViewStream rs = new AddColumnsStream(stream, "fake_col=0");
+ rs = new SumStream(rs, output_columns, new string[] { "fake_col" }, false);
+ rs = new ProjectStream(rs, "!fake_col");
+ mySetRealSource(rs);
+ }
public SumStream(ViewStream stream, string space_seperated_output_columns, string space_seperated_groupby_columns, bool force_null_into_other_columns) {
string[] output_columns = space_seperated_output_columns.Split(' ');
@@ -62,9 +62,9 @@ namespace BCG {
mySetRealSource(rs);
}
- // public SumStream(ViewStream stream, string outputCol)
- // : this(new AddColumnsStream(stream, "fakeCol=0"), outputCol, "fakeCol") {
- // }
+ public SumStream(ViewStream stream, string outputCol)
+ : this(new AddColumnsStream(stream, "fakeCol=0"), outputCol, "fakeCol") {
+ }
}
public class IntSumStream : ViewStream {
diff --git a/src/DataStreamSuite/DataStream/Primitives/Sort/SortStream.cs b/src/DataStreamSuite/DataStream/Primitives/Sort/SortStream.cs
index 7aaf8aff0..3e6926b44 100644
--- a/src/DataStreamSuite/DataStream/Primitives/Sort/SortStream.cs
+++ b/src/DataStreamSuite/DataStream/Primitives/Sort/SortStream.cs
@@ -1,156 +1,99 @@
-// using Bts.ColumnSortExpressions;
-
-// using System.Collections;
-// using System.Collections.Generic;
-// using System.Diagnostics.CodeAnalysis;
-
-// namespace BCG;
-
-// ///
-// /// Defines a Sort stream, with performance of O(n) for all the increments. It uses simple merge sort
-// /// to merge the additions to the accumulates table
-// ///
-// public class SortStream : ViewStream {
-// ViewStream stream;
-// IList sortCols;
-// IComparer? comparer;
-// DataPacket? lastFiredPacket;
-// DataPacket? lastReceivedPacket;
-
-// public SortStream(ViewStream stream, string sortExpression) {
-// this.stream = stream;
-// SortExpression = sortExpression;
-// }
-
-// public SortStream(ViewStream vs, string colName, bool isAsc) {
-// stream = vs;
-// SortExpression = isAsc ? "+" + colName : "-" + colName;
-// }
-
-// public SortStream(ViewStream stream, IComparer comparer) {
-// this.stream = stream;
-// this.comparer = comparer;
-// sortCols = [];
-// }
-
-// [MemberNotNull(nameof(sortCols))]
-// public string SortExpression {
-// set {
-// comparer = null;
-// lastFiredPacket = null;
-// sortCols = ColumnSortExpressionInterpreter.Evaluate(value);
-// if (lastReceivedPacket is not null) {
-// myChangeHandler(lastReceivedPacket);
-// }
-// }
-// }
-
-// public IComparer Comparer {
-// set {
-// comparer = value;
-// lastFiredPacket = null;
-// if (lastReceivedPacket is not null) {
-// myChangeHandler(lastReceivedPacket);
-// }
-// }
-// }
-
-// public void myChangeHandlerTest(object o) =>
-// myChangeHandler(o);
-
-// [MemberNotNull(nameof(lastFiredPacket))]
-// internal void myChangeHandler(object o) {
-// lastReceivedPacket = (DataPacket)o;
-// // First callback, so we need to sort the whole table, this is the
-// // only time we will do that for all the other sorting operations would
-// // be sorting the increments and inserting them into the whole list
-// if (lastFiredPacket is null) {
-// lastFiredPacket = new() {
-// newRows = comparer is null ?
-// new SortedTable(lastReceivedPacket.newRows, sortCols) :
-// new SortedTable(lastReceivedPacket.newRows, comparer)
-// };
-// lastFiredPacket.addedRows = new OrderedBlueTable(lastFiredPacket.newRows);
-// lastFiredPacket.deletedRows = new BlueTable(lastFiredPacket.newRows.schema);
-// myFireEvent(lastFiredPacket.Clone());
-// return;
-// }
-// // Deletes
-// lastFiredPacket.deletedRows =
-// comparer == null ?
-// new SortedTable(lastReceivedPacket.deletedRows, sortCols) :
-// new SortedTable(lastReceivedPacket.deletedRows, comparer);
-// ((SortedTable)lastFiredPacket.newRows).deleteRows(lastFiredPacket.deletedRows);
-// // Adds
-// lastFiredPacket.addedRows = new BlueTable(lastReceivedPacket.addedRows);
-// ((SortedTable)lastFiredPacket.newRows).addRows(lastFiredPacket.addedRows);
-// myFireEvent(lastFiredPacket.Clone());
-// }
-
-// protected override void myOnFirstSubscriber() =>
-// stream.onChange += myChangeHandler;
-
-// protected override void myOnLastSubscriber() {
-// lastFiredPacket = null;
-// lastReceivedPacket = null;
-// stream.onChange -= myChangeHandler;
-// }
-// }
-
-using BCG;
+using Bts.ColumnSortExpressions;
+using System;
using System.Collections;
using System.Collections.Generic;
-using System.Linq;
+using System.Diagnostics.CodeAnalysis;
+namespace BCG;
-[DataStreamUserRespresentation(friendlyName: "sort")]
-public class SortStreamTemp : ViewStream {
- private ViewStream _vs;
- private readonly string _sort_col_name;
- private readonly bool _is_asc = true;
- public SortStreamTemp(ViewStream vs, string sort_exp) {
- _vs = vs;
- if (sort_exp.StartsWith("+")) {
- _is_asc = true;
- _sort_col_name = sort_exp.Substring(1);
- } else if (sort_exp.StartsWith("-")) {
- _is_asc = false;
- _sort_col_name = sort_exp.Substring(1);
- } else {
- _is_asc = true;
- _sort_col_name = sort_exp;
+///
+/// Defines a Sort stream, with performance of O(n) for all the increments. It uses simple merge sort
+/// to merge the additions to the accumulates table
+///
+public class SortStream : ViewStream {
+ ViewStream stream;
+ IList sortCols;
+ IComparer? comparer;
+ DataPacket? lastFiredPacket;
+ DataPacket? lastReceivedPacket;
+
+ public SortStream(ViewStream stream, string sortExpression) {
+ this.stream = stream;
+ SortExpression = sortExpression;
+ }
+
+ public SortStream(ViewStream vs, string colName, bool isAsc) {
+ stream = vs;
+ SortExpression = isAsc ? "+" + colName : "-" + colName;
+ }
+
+ public SortStream(ViewStream stream, IComparer comparer) {
+ this.stream = stream;
+ this.comparer = comparer;
+ sortCols = Array.Empty();
+ }
+
+ [MemberNotNull(nameof(sortCols))]
+ public string SortExpression {
+ set {
+ comparer = null;
+ lastFiredPacket = null;
+ sortCols = ColumnSortExpressionInterpreter.Evaluate(value);
+ if (lastReceivedPacket is not null) {
+ myChangeHandler(lastReceivedPacket);
+ }
}
}
- protected override void myOnFirstSubscriber() {
- _vs.onChange += _handle_vs_update;
+ public IComparer Comparer {
+ set {
+ comparer = value;
+ lastFiredPacket = null;
+ if (lastReceivedPacket is not null) {
+ myChangeHandler(lastReceivedPacket);
+ }
+ }
}
+ public void myChangeHandlerTest(object o) =>
+ myChangeHandler(o);
+
+ [MemberNotNull(nameof(lastFiredPacket))]
+ internal void myChangeHandler(object o) {
+ lastReceivedPacket = (DataPacket)o;
+ // First callback, so we need to sort the whole table, this is the
+ // only time we will do that for all the other sorting operations would
+ // be sorting the increments and inserting them into the whole list
+ if (lastFiredPacket is null) {
+ lastFiredPacket = new() {
+ newRows = comparer is null ?
+ new SortedTable(lastReceivedPacket.newRows, sortCols) :
+ new SortedTable(lastReceivedPacket.newRows, comparer)
+ };
+ lastFiredPacket.addedRows = new OrderedBlueTable(lastFiredPacket.newRows);
+ lastFiredPacket.deletedRows = new BlueTable(lastFiredPacket.newRows.schema);
+ myFireEvent(lastFiredPacket.Clone());
+ return;
+ }
+ // Deletes
+ lastFiredPacket.deletedRows =
+ comparer == null ?
+ new SortedTable(lastReceivedPacket.deletedRows, sortCols) :
+ new SortedTable(lastReceivedPacket.deletedRows, comparer);
+ ((SortedTable)lastFiredPacket.newRows).deleteRows(lastFiredPacket.deletedRows);
+ // Adds
+ lastFiredPacket.addedRows = new BlueTable(lastReceivedPacket.addedRows);
+ ((SortedTable)lastFiredPacket.newRows).addRows(lastFiredPacket.addedRows);
+ myFireEvent(lastFiredPacket.Clone());
+ }
+
+ protected override void myOnFirstSubscriber() =>
+ stream.onChange += myChangeHandler;
+
protected override void myOnLastSubscriber() {
- _vs.onChange -= _handle_vs_update;
+ lastFiredPacket = null;
+ lastReceivedPacket = null;
+ stream.onChange -= myChangeHandler;
}
-
- private void _handle_vs_update(object dp_obj) {
- var dp = dp_obj as DataPacket;
- IBlueRowCollection nrows = dp.newRows;
- var _out_schema = nrows.schema;
- List row_list = new List();
- foreach (IBlueRow r in nrows)
- row_list.Add(r);
- //OrderBy sorts in ascedning order
- var sorted_rows = row_list.OrderBy(r => r[_sort_col_name]).ToList();
- if (!_is_asc)
- // reverse it to make it descending
- sorted_rows.Reverse();
- //sort rows so we can do the diff
- OrderedBlueTable rv = new OrderedBlueTable(_out_schema);
- foreach (IBlueRow r in sorted_rows) {
- rv.addRow(r);
- }
-
- var dp_out = BlueDataUtility.build_dp_inc(new_rows: rv, myLastValue as DataPacket, true);
- myFireEvent(dp_out);
-
- }
-}
\ No newline at end of file
+}
diff --git a/src/DataStreamSuite/DataStream/Primitives/Sort/SortedTable.cs b/src/DataStreamSuite/DataStream/Primitives/Sort/SortedTable.cs
index 51252b16d..77dfecd42 100644
--- a/src/DataStreamSuite/DataStream/Primitives/Sort/SortedTable.cs
+++ b/src/DataStreamSuite/DataStream/Primitives/Sort/SortedTable.cs
@@ -11,9 +11,9 @@ namespace BCG;
/// Represents an IBlueRowCollection with the rows sorted according to the
///
public class SortedTable : IBlueRowCollection {
- readonly List rows = [];
- readonly Dictionary rowIndex = [];
- readonly IList sortCols = [];
+ readonly List rows = new(0);
+ readonly Dictionary rowIndex = new(0);
+ readonly IList sortCols = Array.Empty();
readonly IComparer? comparer;
///
diff --git a/src/DataStreamSuite/DataStream/Primitives/Top/TopStream.cs b/src/DataStreamSuite/DataStream/Primitives/Top/TopStream.cs
index 10b68a4ad..c67dbd82c 100644
--- a/src/DataStreamSuite/DataStream/Primitives/Top/TopStream.cs
+++ b/src/DataStreamSuite/DataStream/Primitives/Top/TopStream.cs
@@ -1,49 +1,43 @@
-// using System;
+using System;
-// namespace BCG
-// {
-// public class TopStream : MethodDrivenStream
-// {
-// private int myTopCount;
+namespace BCG {
+ public class TopStream : MethodDrivenStream {
+ private int myTopCount;
-// public TopStream (ViewStream stream, int topCount) : base (stream) {myTopCount = topCount;}
+ public TopStream(ViewStream stream, int topCount) : base(stream) { myTopCount = topCount; }
-// public TopStream (ViewStream stream,
-// int topCount,
-// string sortExpression)
-// :
-// this (sortExpression == null?
-// stream :
-// new SortStream (stream, sortExpression), topCount)
-// {}
+ public TopStream(ViewStream stream,
+ int topCount,
+ string sortExpression)
+ :
+ this(sortExpression == null ?
+ stream :
+ new SortStream(stream, sortExpression), topCount) { }
-// protected override void myOnTableClear ()
-// {
-// myLastFiredPacket.deletedRows = myLastFiredPacket.newRows;
-// myLastFiredPacket.newRows = new OrderedBlueTable (myLastFiredPacket.deletedRows.schema);
-// myLastFiredPacket.addedRows = new OrderedBlueTable (myLastFiredPacket.deletedRows.schema);
-// }
+ protected override void myOnTableClear() {
+ myLastFiredPacket.deletedRows = myLastFiredPacket.newRows;
+ myLastFiredPacket.newRows = new OrderedBlueTable(myLastFiredPacket.deletedRows.schema);
+ myLastFiredPacket.addedRows = new OrderedBlueTable(myLastFiredPacket.deletedRows.schema);
+ }
-// protected override void myHandleFirstCall ()
-// {
-// myLastFiredPacket = new DataPacket ();
-// myLastFiredPacket.newRows = new OrderedBlueTable (myLastArrivedPacket.newRows, myTopCount);
-// myLastFiredPacket.addedRows = new OrderedBlueTable(myLastFiredPacket.newRows);
-// myLastFiredPacket.deletedRows = new OrderedBlueTable (myLastFiredPacket.newRows.schema);
-// }
+ protected override void myHandleFirstCall() {
+ myLastFiredPacket = new DataPacket();
+ myLastFiredPacket.newRows = new OrderedBlueTable(myLastArrivedPacket.newRows, myTopCount);
+ myLastFiredPacket.addedRows = new OrderedBlueTable(myLastFiredPacket.newRows);
+ myLastFiredPacket.deletedRows = new OrderedBlueTable(myLastFiredPacket.newRows.schema);
+ }
-// protected override void myHandleIncrementalAdd () {}
+ protected override void myHandleIncrementalAdd() { }
-// protected override void myHandleIncrementalDelete()
-// {
-// IBlueRowCollection oldRows = myLastFiredPacket.newRows;
-// myLastFiredPacket.newRows = new OrderedBlueTable (myLastArrivedPacket.newRows, myTopCount);
-// BlueDataUtility.computeDifferences (myLastFiredPacket.newRows,
-// oldRows,
-// myLastFiredPacket.schema.columnNames,
-// out myLastFiredPacket.deletedRows,
-// out myLastFiredPacket.addedRows);
-// myFireSwitch = myLastFiredPacket.addedRows.Count + myLastFiredPacket.deletedRows.Count == 0 ? false : true;
-// }
-// }
-// }
\ No newline at end of file
+ protected override void myHandleIncrementalDelete() {
+ IBlueRowCollection oldRows = myLastFiredPacket.newRows;
+ myLastFiredPacket.newRows = new OrderedBlueTable(myLastArrivedPacket.newRows, myTopCount);
+ BlueDataUtility.computeDifferences(myLastFiredPacket.newRows,
+ oldRows,
+ myLastFiredPacket.schema.columnNames,
+ out myLastFiredPacket.deletedRows,
+ out myLastFiredPacket.addedRows);
+ myFireSwitch = myLastFiredPacket.addedRows.Count + myLastFiredPacket.deletedRows.Count == 0 ? false : true;
+ }
+ }
+}
\ No newline at end of file
diff --git a/src/DataStreamSuite/DataStreamExpressions/GplexBuffers.cs b/src/DataStreamSuite/DataStreamExpressions/GplexBuffers.cs
index 76dd35443..1bcc08e09 100644
--- a/src/DataStreamSuite/DataStreamExpressions/GplexBuffers.cs
+++ b/src/DataStreamSuite/DataStreamExpressions/GplexBuffers.cs
@@ -1,720 +1,720 @@
-// ==============================================================
-//
-// This code automatically produced from an embedded resource.
-// Do not edit this file, or it will become incompatible with
-// the specification from which it was generated.
-//
-// ==============================================================
-
-using System;
-using System.IO;
-using System.Text;
-using System.Collections.Generic;
-using System.Diagnostics.CodeAnalysis;
-using System.Runtime.Serialization;
-using System.Globalization;
-
-namespace QUT.GplexBuffers
-{
-// Code copied from GPLEX embedded resource
- [Serializable]
- internal class BufferException : Exception
- {
- public BufferException() { }
- public BufferException(string message) : base(message) { }
- public BufferException(string message, Exception innerException)
- : base(message, innerException) { }
- protected BufferException(SerializationInfo info, StreamingContext context)
- : base(info, context) { }
- }
-
- internal abstract class ScanBuff
- {
- private string fileNm;
-
- public const int EndOfFile = -1;
- public const int UnicodeReplacementChar = 0xFFFD;
-
- public bool IsFile { get { return (fileNm != null); } }
- public string FileName { get { return fileNm; } set { fileNm = value; } }
-
- public abstract int Pos { get; set; }
- public abstract int Read();
- public virtual void Mark() { }
-
- public abstract string GetString(int begin, int limit);
-
- public static ScanBuff GetBuffer(string source)
- {
- return new StringBuffer(source);
- }
-
- public static ScanBuff GetBuffer(IList source)
- {
- return new LineBuffer(source);
- }
-
-#if (!NOFILES)
- public static ScanBuff GetBuffer(Stream source)
- {
- return new BuildBuffer(source);
- }
-
-#if (!BYTEMODE)
- public static ScanBuff GetBuffer(Stream source, int fallbackCodePage)
- {
- return new BuildBuffer(source, fallbackCodePage);
- }
-#endif // !BYTEMODE
-#endif // !NOFILES
- }
-
- #region Buffer classes
-
- // ==============================================================
- // ===== Definitions for various ScanBuff derived classes ====
- // ==============================================================
- // =============== String input ================
- // ==============================================================
-
- ///
- /// This class reads characters from a single string as
- /// required, for example, by Visual Studio language services
- ///
- sealed class StringBuffer : ScanBuff
- {
- string str; // input buffer
- int bPos; // current position in buffer
- int sLen;
-
- public StringBuffer(string source)
- {
- this.str = source;
- this.sLen = source.Length;
- this.FileName = null;
- }
-
- public override int Read()
- {
- if (bPos < sLen) return str[bPos++];
- else if (bPos == sLen) { bPos++; return '\n'; } // one strike, see new line
- else { bPos++; return EndOfFile; } // two strikes and you're out!
- }
-
- public override string GetString(int begin, int limit)
- {
- // "limit" can be greater than sLen with the BABEL
- // option set. Read returns a "virtual" EOL if
- // an attempt is made to read past the end of the
- // string buffer. Without the guard any attempt
- // to fetch yytext for a token that includes the
- // EOL will throw an index exception.
- if (limit > sLen) limit = sLen;
- if (limit <= begin) return "";
- else return str.Substring(begin, limit - begin);
- }
-
- public override int Pos
- {
- get { return bPos; }
- set { bPos = value; }
- }
-
- public override string ToString() { return "StringBuffer"; }
- }
-
- // ==============================================================
- // The LineBuff class contributed by Nigel Horspool,
- // nigelh@cs.uvic.cs
- // ==============================================================
-
- sealed class LineBuffer : ScanBuff
- {
- IList line; // list of source lines from a file
- int numLines; // number of strings in line list
- string curLine; // current line in that list
- int cLine; // index of current line in the list
- int curLen; // length of current line
- int curLineStart; // position of line start in whole file
- int curLineEnd; // position of line end in whole file
- int maxPos; // max position ever visited in whole file
- int cPos; // ordinal number of code in source
-
- // Constructed from a list of strings, one per source line.
- // The lines have had trailing '\n' characters removed.
- public LineBuffer(IList lineList)
- {
- line = lineList;
- numLines = line.Count;
- cPos = curLineStart = 0;
- curLine = (numLines > 0 ? line[0] : "");
- maxPos = curLineEnd = curLen = curLine.Length;
- cLine = 1;
- FileName = null;
- }
-
- public override int Read()
- {
- if (cPos < curLineEnd)
- return curLine[cPos++ - curLineStart];
- if (cPos++ == curLineEnd)
- return '\n';
- if (cLine >= numLines)
- return EndOfFile;
- curLine = line[cLine];
- curLen = curLine.Length;
- curLineStart = curLineEnd + 1;
- curLineEnd = curLineStart + curLen;
- if (curLineEnd > maxPos)
- maxPos = curLineEnd;
- cLine++;
- return curLen > 0 ? curLine[0] : '\n';
- }
-
- // To speed up searches for the line containing a position
- private int cachedPosition;
- private int cachedIxdex;
- private int cachedLineStart;
-
- // Given a position pos within the entire source, the results are
- // ix -- the index of the containing line
- // lstart -- the position of the first character on that line
- private void findIndex(int pos, out int ix, out int lstart)
- {
- if (pos >= cachedPosition)
- {
- ix = cachedIxdex; lstart = cachedLineStart;
- }
- else
- {
- ix = lstart = 0;
- }
- while (ix < numLines)
- {
- int len = line[ix].Length + 1;
- if (pos < lstart + len) break;
- lstart += len;
- ix++;
- }
- cachedPosition = pos;
- cachedIxdex = ix;
- cachedLineStart = lstart;
- }
-
- public override string GetString(int begin, int limit)
- {
- if (begin >= maxPos || limit <= begin) return "";
- int endIx, begIx, endLineStart, begLineStart;
- findIndex(begin, out begIx, out begLineStart);
- int begCol = begin - begLineStart;
- findIndex(limit, out endIx, out endLineStart);
- int endCol = limit - endLineStart;
- string s = line[begIx];
- if (begIx == endIx)
- {
- // the usual case, substring all on one line
- return (endCol <= s.Length) ?
- s.Substring(begCol, endCol - begCol)
- : s.Substring(begCol) + "\n";
- }
- // the string spans multiple lines, yuk!
- StringBuilder sb = new StringBuilder();
- if (begCol < s.Length)
- sb.Append(s.Substring(begCol));
- for (; ; )
- {
- sb.Append("\n");
- s = line[++begIx];
- if (begIx >= endIx) break;
- sb.Append(s);
- }
- if (endCol <= s.Length)
- {
- sb.Append(s.Substring(0, endCol));
- }
- else
- {
- sb.Append(s);
- sb.Append("\n");
- }
- return sb.ToString();
- }
-
- public override int Pos
- {
- get { return cPos; }
- set
- {
- cPos = value;
- findIndex(cPos, out cLine, out curLineStart);
- // cLine should be the *next* line after curLine.
- curLine = (cLine < numLines ? line[cLine++] : "");
- curLineEnd = curLineStart + curLine.Length;
- }
- }
-
- public override string ToString() { return "LineBuffer"; }
- }
-
-#if (!NOFILES)
- // ==============================================================
- // ===== class BuildBuff : for unicode text files ========
- // ==============================================================
-
- class BuildBuffer : ScanBuff
- {
- // Double buffer for char stream.
- class BufferElement
- {
- StringBuilder bldr = new StringBuilder();
- StringBuilder next = new StringBuilder();
- int minIx;
- int maxIx;
- int brkIx;
- bool appendToNext;
-
- internal BufferElement() { }
-
- internal int MaxIndex { get { return maxIx; } }
- // internal int MinIndex { get { return minIx; } }
-
- internal char this[int index]
- {
- get
- {
- if (index < minIx || index >= maxIx)
- throw new BufferException("Index was outside data buffer");
- else if (index < brkIx)
- return bldr[index - minIx];
- else
- return next[index - brkIx];
- }
- }
-
- internal void Append(char[] block, int count)
- {
- maxIx += count;
- if (appendToNext)
- this.next.Append(block, 0, count);
- else
- {
- this.bldr.Append(block, 0, count);
- brkIx = maxIx;
- appendToNext = true;
- }
- }
-
- internal string GetString(int start, int limit)
- {
- if (limit <= start)
- return "";
- if (start >= minIx && limit <= maxIx)
- if (limit < brkIx) // String entirely in bldr builder
- return bldr.ToString(start - minIx, limit - start);
- else if (start >= brkIx) // String entirely in next builder
- return next.ToString(start - brkIx, limit - start);
- else // Must do a string-concatenation
- return
- bldr.ToString(start - minIx, brkIx - start) +
- next.ToString(0, limit - brkIx);
- else
- throw new BufferException("String was outside data buffer");
- }
-
- internal void Mark(int limit)
- {
- if (limit > brkIx + 16) // Rotate blocks
- {
- StringBuilder temp = bldr;
- bldr = next;
- next = temp;
- next.Length = 0;
- minIx = brkIx;
- brkIx = maxIx;
- }
- }
- }
-
- BufferElement data = new BufferElement();
-
- int bPos; // Postion index in the StringBuilder
- BlockReader NextBlk; // Delegate that serves char-arrays;
-
- private string EncodingName
- {
- get
- {
- StreamReader rdr = NextBlk.Target as StreamReader;
- return (rdr == null ? "raw-bytes" : rdr.CurrentEncoding.BodyName);
- }
- }
-
- public BuildBuffer(Stream stream)
- {
- FileStream fStrm = (stream as FileStream);
- if (fStrm != null) FileName = fStrm.Name;
- NextBlk = BlockReaderFactory.Raw(stream);
- }
-
-#if (!BYTEMODE)
- public BuildBuffer(Stream stream, int fallbackCodePage)
- {
- FileStream fStrm = (stream as FileStream);
- if (fStrm != null) FileName = fStrm.Name;
- NextBlk = BlockReaderFactory.Get(stream, fallbackCodePage);
- }
-#endif
-
- ///
- /// Marks a conservative lower bound for the buffer,
- /// allowing space to be reclaimed. If an application
- /// needs to call GetString at arbitrary past locations
- /// in the input stream, Mark() is not called.
- ///
- public override void Mark() { data.Mark(bPos - 2); }
-
- public override int Pos
- {
- get { return bPos; }
- set { bPos = value; }
- }
-
-
- ///
- /// Read returns the ordinal number of the next char, or
- /// EOF (-1) for an end of stream. Note that the next
- /// code point may require *two* calls of Read().
- ///
- ///
- public override int Read()
- {
- //
- // Characters at positions
- // [data.offset, data.offset + data.bldr.Length)
- // are available in data.bldr.
- //
- if (bPos < data.MaxIndex)
- {
- // ch0 cannot be EOF
- return (int)data[bPos++];
- }
- else // Read from underlying stream
- {
- // Experimental code, blocks of page size
- char[] chrs = new char[4096];
- int count = NextBlk(chrs, 0, 4096);
- if (count == 0)
- return EndOfFile;
- else
- {
- data.Append(chrs, count);
- return (int)data[bPos++];
- }
- }
- }
-
- public override string GetString(int begin, int limit)
- {
- return data.GetString(begin, limit);
- }
-
- public override string ToString()
- {
- return "StringBuilder buffer, encoding: " + this.EncodingName;
- }
- }
-
- // =============== End ScanBuff-derived classes ==================
-
- internal delegate int BlockReader(char[] block, int index, int number);
-
- // A delegate factory, serving up a delegate that
- // reads a block of characters from the underlying
- // encoded stream, via a StreamReader object.
- //
- internal static class BlockReaderFactory
- {
- public static BlockReader Raw(Stream stream)
- {
- return delegate(char[] block, int index, int number)
- {
- byte[] b = new byte[number];
- int count = stream.Read(b, 0, number);
- int i = 0;
- int j = index;
- for (; i < count; i++, j++)
- block[j] = (char)b[i];
- return count;
- };
- }
-
-#if (!BYTEMODE)
- public static BlockReader Get(Stream stream, int fallbackCodePage)
- {
- Encoding encoding;
- int preamble = Preamble(stream);
-
- if (preamble != 0) // There is a valid BOM here!
- encoding = Encoding.GetEncoding(preamble);
- else if (fallbackCodePage == -1) // Fallback is "raw" bytes
- return Raw(stream);
- else if (fallbackCodePage != -2) // Anything but "guess"
- encoding = Encoding.GetEncoding(fallbackCodePage);
- else // This is the "guess" option
- {
- int guess = new Guesser(stream).GuessCodePage();
- stream.Seek(0, SeekOrigin.Begin);
- if (guess == -1) // ==> this is a 7-bit file
- encoding = Encoding.ASCII;
- else if (guess == 65001)
- encoding = Encoding.UTF8;
- else // ==> use the machine default
- encoding = Encoding.Default;
- }
- StreamReader reader = new StreamReader(stream, encoding);
- return reader.Read;
- }
-
- static int Preamble(Stream stream)
- {
- int b0 = stream.ReadByte();
- int b1 = stream.ReadByte();
-
- if (b0 == 0xfe && b1 == 0xff)
- return 1201; // UTF16BE
- if (b0 == 0xff && b1 == 0xfe)
- return 1200; // UTF16LE
-
- int b2 = stream.ReadByte();
- if (b0 == 0xef && b1 == 0xbb && b2 == 0xbf)
- return 65001; // UTF8
- //
- // There is no unicode preamble, so we
- // return denoter for the machine default.
- //
- stream.Seek(0, SeekOrigin.Begin);
- return 0;
- }
-#endif // !BYTEMODE
- }
-#endif // !NOFILES
- #endregion Buffer classes
-
- // ==============================================================
- // ============ class CodePageHandling =============
- // ==============================================================
-#if (!NOFILES)
- public static class CodePageHandling
- {
- public static int GetCodePage(string option)
- {
- string command = option.ToUpperInvariant();
- if (command.StartsWith("CodePage:", StringComparison.OrdinalIgnoreCase))
- command = command.Substring(9);
- try
- {
- if (command.Equals("RAW"))
- return -1;
- else if (command.Equals("GUESS"))
- return -2;
- else if (command.Equals("DEFAULT"))
- return 0;
- else if (char.IsDigit(command[0]))
- return int.Parse(command, CultureInfo.InvariantCulture);
- else
- {
- Encoding enc = Encoding.GetEncoding(command);
- return enc.CodePage;
- }
- }
- catch (FormatException)
- {
- Console.Error.WriteLine(
- "Invalid format \"{0}\", using machine default", option);
- }
- catch (ArgumentException)
- {
- Console.Error.WriteLine(
- "Unknown code page \"{0}\", using machine default", option);
- }
- return 0;
- }
- }
-#region guesser
-#if (!BYTEMODE)
- // ==============================================================
- // ============ Encoding Guesser =============
- // ==============================================================
-
- ///
- /// This class provides a simple finite state automaton that
- /// scans the file looking for (1) valid UTF-8 byte patterns,
- /// (2) bytes >= 0x80 which are not part of a UTF-8 sequence.
- /// The method then guesses whether it is UTF-8 or maybe some
- /// local machine default encoding. This works well for the
- /// various Latin encodings.
- ///
- internal class Guesser
- {
- ScanBuff buffer;
-
- public int GuessCodePage() { return Scan(); }
-
- const int maxAccept = 10;
- const int initial = 0;
- const int eofNum = 0;
- const int goStart = -1;
- const int INITIAL = 0;
- const int EndToken = 0;
-
- #region user code
- /*
- * Reads the bytes of a file to determine if it is
- * UTF-8 or a single-byte code page file.
- */
- public long utfX;
- public long uppr;
- #endregion user code
-
- int state;
- int currentStart = startState[0];
- int code;
-
- #region ScannerTables
- static int[] startState = new int[] { 11, 0 };
-
- #region CharacterMap
- static sbyte[] map = new sbyte[256] {
-/* '\0' */ 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0,
-/* '\x10' */ 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0,
-/* '\x20' */ 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0,
-/* '0' */ 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0,
-/* '@' */ 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0,
-/* 'P' */ 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0,
-/* '`' */ 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0,
-/* 'p' */ 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0,
-/* '\x80' */ 2, 2, 2, 2, 2, 2, 2, 2, 2, 2, 2, 2, 2, 2, 2, 2,
-/* '\x90' */ 2, 2, 2, 2, 2, 2, 2, 2, 2, 2, 2, 2, 2, 2, 2, 2,
-/* '\xA0' */ 2, 2, 2, 2, 2, 2, 2, 2, 2, 2, 2, 2, 2, 2, 2, 2,
-/* '\xB0' */ 2, 2, 2, 2, 2, 2, 2, 2, 2, 2, 2, 2, 2, 2, 2, 2,
-/* '\xC0' */ 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1,
-/* '\xD0' */ 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1,
-/* '\xE0' */ 3, 3, 3, 3, 3, 3, 3, 3, 3, 3, 3, 3, 3, 3, 3, 3,
-/* '\xF0' */ 4, 4, 4, 4, 4, 4, 4, 4, 5, 5, 5, 5, 5, 5, 5, 5 };
- #endregion
-
- static sbyte[][] nextState = new sbyte[][] {
- new sbyte[] {0, 0, 0, 0, 0, 0},
- new sbyte[] {-1, -1, 10, -1, -1, -1},
- new sbyte[] {-1, -1, -1, -1, -1, -1},
- new sbyte[] {-1, -1, 8, -1, -1, -1},
- new sbyte[] {-1, -1, 5, -1, -1, -1},
- new sbyte[] {-1, -1, 6, -1, -1, -1},
- new sbyte[] {-1, -1, 7, -1, -1, -1},
- null,
- new sbyte[] {-1, -1, 9, -1, -1, -1},
- null,
- null,
- new sbyte[] {-1, 1, 2, 3, 4, 2}
- };
-
-
- [SuppressMessage("Microsoft.Performance", "CA1810:InitializeReferenceTypeStaticFieldsInline")]
- // Reason for suppression: cannot have self-reference in array initializer.
- static Guesser()
- {
- nextState[7] = nextState[2];
- nextState[9] = nextState[2];
- nextState[10] = nextState[2];
- }
-
- int NextState()
- {
- if (code == ScanBuff.EndOfFile)
- return eofNum;
- else
- return nextState[state][map[code]];
- }
- #endregion
-
- public Guesser(System.IO.Stream file) { SetSource(file); }
-
- public void SetSource(System.IO.Stream source)
- {
- this.buffer = new BuildBuffer(source);
- code = buffer.Read();
- }
-
- int Scan()
- {
- for (; ; )
- {
- int next;
- state = currentStart;
- while ((next = NextState()) == goStart)
- code = buffer.Read();
-
- state = next;
- code = buffer.Read();
-
- while ((next = NextState()) > eofNum)
- {
- state = next;
- code = buffer.Read();
- }
- if (state <= maxAccept)
- {
- #region ActionSwitch
-#pragma warning disable 162
- switch (state)
- {
- case eofNum:
- switch (currentStart)
- {
- case 11:
- if (utfX == 0 && uppr == 0) return -1; /* raw ascii */
- else if (uppr * 10 > utfX) return 0; /* default code page */
- else return 65001; /* UTF-8 encoding */
- break;
- }
- return EndToken;
- case 1: // Recognized '{Upper128}', Shortest string "\xC0"
- case 2: // Recognized '{Upper128}', Shortest string "\x80"
- case 3: // Recognized '{Upper128}', Shortest string "\xE0"
- case 4: // Recognized '{Upper128}', Shortest string "\xF0"
- uppr++;
- break;
- case 5: // Recognized '{Utf8pfx4}{Utf8cont}', Shortest string "\xF0\x80"
- uppr += 2;
- break;
- case 6: // Recognized '{Utf8pfx4}{Utf8cont}{2}', Shortest string "\xF0\x80\x80"
- uppr += 3;
- break;
- case 7: // Recognized '{Utf8pfx4}{Utf8cont}{3}', Shortest string "\xF0\x80\x80\x80"
- utfX += 3;
- break;
- case 8: // Recognized '{Utf8pfx3}{Utf8cont}', Shortest string "\xE0\x80"
- uppr += 2;
- break;
- case 9: // Recognized '{Utf8pfx3}{Utf8cont}{2}', Shortest string "\xE0\x80\x80"
- utfX += 2;
- break;
- case 10: // Recognized '{Utf8pfx2}{Utf8cont}', Shortest string "\xC0\x80"
- utfX++;
- break;
- default:
- break;
- }
-#pragma warning restore 162
- #endregion
- }
- }
- }
- } // end class Guesser
-
-#endif // !BYTEMODE
-#endregion
-#endif // !NOFILES
-
-// End of code copied from embedded resource
-}
+// ==============================================================
+//
+// This code automatically produced from an embedded resource.
+// Do not edit this file, or it will become incompatible with
+// the specification from which it was generated.
+//
+// ==============================================================
+
+using System;
+using System.IO;
+using System.Text;
+using System.Collections.Generic;
+using System.Diagnostics.CodeAnalysis;
+using System.Runtime.Serialization;
+using System.Globalization;
+
+namespace QUT.GplexBuffers
+{
+// Code copied from GPLEX embedded resource
+ [Serializable]
+ internal class BufferException : Exception
+ {
+ public BufferException() { }
+ public BufferException(string message) : base(message) { }
+ public BufferException(string message, Exception innerException)
+ : base(message, innerException) { }
+ protected BufferException(SerializationInfo info, StreamingContext context)
+ : base(info, context) { }
+ }
+
+ internal abstract class ScanBuff
+ {
+ private string fileNm;
+
+ public const int EndOfFile = -1;
+ public const int UnicodeReplacementChar = 0xFFFD;
+
+ public bool IsFile { get { return (fileNm != null); } }
+ public string FileName { get { return fileNm; } set { fileNm = value; } }
+
+ public abstract int Pos { get; set; }
+ public abstract int Read();
+ public virtual void Mark() { }
+
+ public abstract string GetString(int begin, int limit);
+
+ public static ScanBuff GetBuffer(string source)
+ {
+ return new StringBuffer(source);
+ }
+
+ public static ScanBuff GetBuffer(IList source)
+ {
+ return new LineBuffer(source);
+ }
+
+#if (!NOFILES)
+ public static ScanBuff GetBuffer(Stream source)
+ {
+ return new BuildBuffer(source);
+ }
+
+#if (!BYTEMODE)
+ public static ScanBuff GetBuffer(Stream source, int fallbackCodePage)
+ {
+ return new BuildBuffer(source, fallbackCodePage);
+ }
+#endif // !BYTEMODE
+#endif // !NOFILES
+ }
+
+ #region Buffer classes
+
+ // ==============================================================
+ // ===== Definitions for various ScanBuff derived classes ====
+ // ==============================================================
+ // =============== String input ================
+ // ==============================================================
+
+ ///
+ /// This class reads characters from a single string as
+ /// required, for example, by Visual Studio language services
+ ///
+ sealed class StringBuffer : ScanBuff
+ {
+ string str; // input buffer
+ int bPos; // current position in buffer
+ int sLen;
+
+ public StringBuffer(string source)
+ {
+ this.str = source;
+ this.sLen = source.Length;
+ this.FileName = null;
+ }
+
+ public override int Read()
+ {
+ if (bPos < sLen) return str[bPos++];
+ else if (bPos == sLen) { bPos++; return '\n'; } // one strike, see new line
+ else { bPos++; return EndOfFile; } // two strikes and you're out!
+ }
+
+ public override string GetString(int begin, int limit)
+ {
+ // "limit" can be greater than sLen with the BABEL
+ // option set. Read returns a "virtual" EOL if
+ // an attempt is made to read past the end of the
+ // string buffer. Without the guard any attempt
+ // to fetch yytext for a token that includes the
+ // EOL will throw an index exception.
+ if (limit > sLen) limit = sLen;
+ if (limit <= begin) return "";
+ else return str.Substring(begin, limit - begin);
+ }
+
+ public override int Pos
+ {
+ get { return bPos; }
+ set { bPos = value; }
+ }
+
+ public override string ToString() { return "StringBuffer"; }
+ }
+
+ // ==============================================================
+ // The LineBuff class contributed by Nigel Horspool,
+ // nigelh@cs.uvic.cs
+ // ==============================================================
+
+ sealed class LineBuffer : ScanBuff
+ {
+ IList line; // list of source lines from a file
+ int numLines; // number of strings in line list
+ string curLine; // current line in that list
+ int cLine; // index of current line in the list
+ int curLen; // length of current line
+ int curLineStart; // position of line start in whole file
+ int curLineEnd; // position of line end in whole file
+ int maxPos; // max position ever visited in whole file
+ int cPos; // ordinal number of code in source
+
+ // Constructed from a list of strings, one per source line.
+ // The lines have had trailing '\n' characters removed.
+ public LineBuffer(IList lineList)
+ {
+ line = lineList;
+ numLines = line.Count;
+ cPos = curLineStart = 0;
+ curLine = (numLines > 0 ? line[0] : "");
+ maxPos = curLineEnd = curLen = curLine.Length;
+ cLine = 1;
+ FileName = null;
+ }
+
+ public override int Read()
+ {
+ if (cPos < curLineEnd)
+ return curLine[cPos++ - curLineStart];
+ if (cPos++ == curLineEnd)
+ return '\n';
+ if (cLine >= numLines)
+ return EndOfFile;
+ curLine = line[cLine];
+ curLen = curLine.Length;
+ curLineStart = curLineEnd + 1;
+ curLineEnd = curLineStart + curLen;
+ if (curLineEnd > maxPos)
+ maxPos = curLineEnd;
+ cLine++;
+ return curLen > 0 ? curLine[0] : '\n';
+ }
+
+ // To speed up searches for the line containing a position
+ private int cachedPosition;
+ private int cachedIxdex;
+ private int cachedLineStart;
+
+ // Given a position pos within the entire source, the results are
+ // ix -- the index of the containing line
+ // lstart -- the position of the first character on that line
+ private void findIndex(int pos, out int ix, out int lstart)
+ {
+ if (pos >= cachedPosition)
+ {
+ ix = cachedIxdex; lstart = cachedLineStart;
+ }
+ else
+ {
+ ix = lstart = 0;
+ }
+ while (ix < numLines)
+ {
+ int len = line[ix].Length + 1;
+ if (pos < lstart + len) break;
+ lstart += len;
+ ix++;
+ }
+ cachedPosition = pos;
+ cachedIxdex = ix;
+ cachedLineStart = lstart;
+ }
+
+ public override string GetString(int begin, int limit)
+ {
+ if (begin >= maxPos || limit <= begin) return "";
+ int endIx, begIx, endLineStart, begLineStart;
+ findIndex(begin, out begIx, out begLineStart);
+ int begCol = begin - begLineStart;
+ findIndex(limit, out endIx, out endLineStart);
+ int endCol = limit - endLineStart;
+ string s = line[begIx];
+ if (begIx == endIx)
+ {
+ // the usual case, substring all on one line
+ return (endCol <= s.Length) ?
+ s.Substring(begCol, endCol - begCol)
+ : s.Substring(begCol) + "\n";
+ }
+ // the string spans multiple lines, yuk!
+ StringBuilder sb = new StringBuilder();
+ if (begCol < s.Length)
+ sb.Append(s.Substring(begCol));
+ for (; ; )
+ {
+ sb.Append("\n");
+ s = line[++begIx];
+ if (begIx >= endIx) break;
+ sb.Append(s);
+ }
+ if (endCol <= s.Length)
+ {
+ sb.Append(s.Substring(0, endCol));
+ }
+ else
+ {
+ sb.Append(s);
+ sb.Append("\n");
+ }
+ return sb.ToString();
+ }
+
+ public override int Pos
+ {
+ get { return cPos; }
+ set
+ {
+ cPos = value;
+ findIndex(cPos, out cLine, out curLineStart);
+ // cLine should be the *next* line after curLine.
+ curLine = (cLine < numLines ? line[cLine++] : "");
+ curLineEnd = curLineStart + curLine.Length;
+ }
+ }
+
+ public override string ToString() { return "LineBuffer"; }
+ }
+
+#if (!NOFILES)
+ // ==============================================================
+ // ===== class BuildBuff : for unicode text files ========
+ // ==============================================================
+
+ class BuildBuffer : ScanBuff
+ {
+ // Double buffer for char stream.
+ class BufferElement
+ {
+ StringBuilder bldr = new StringBuilder();
+ StringBuilder next = new StringBuilder();
+ int minIx;
+ int maxIx;
+ int brkIx;
+ bool appendToNext;
+
+ internal BufferElement() { }
+
+ internal int MaxIndex { get { return maxIx; } }
+ // internal int MinIndex { get { return minIx; } }
+
+ internal char this[int index]
+ {
+ get
+ {
+ if (index < minIx || index >= maxIx)
+ throw new BufferException("Index was outside data buffer");
+ else if (index < brkIx)
+ return bldr[index - minIx];
+ else
+ return next[index - brkIx];
+ }
+ }
+
+ internal void Append(char[] block, int count)
+ {
+ maxIx += count;
+ if (appendToNext)
+ this.next.Append(block, 0, count);
+ else
+ {
+ this.bldr.Append(block, 0, count);
+ brkIx = maxIx;
+ appendToNext = true;
+ }
+ }
+
+ internal string GetString(int start, int limit)
+ {
+ if (limit <= start)
+ return "";
+ if (start >= minIx && limit <= maxIx)
+ if (limit < brkIx) // String entirely in bldr builder
+ return bldr.ToString(start - minIx, limit - start);
+ else if (start >= brkIx) // String entirely in next builder
+ return next.ToString(start - brkIx, limit - start);
+ else // Must do a string-concatenation
+ return
+ bldr.ToString(start - minIx, brkIx - start) +
+ next.ToString(0, limit - brkIx);
+ else
+ throw new BufferException("String was outside data buffer");
+ }
+
+ internal void Mark(int limit)
+ {
+ if (limit > brkIx + 16) // Rotate blocks
+ {
+ StringBuilder temp = bldr;
+ bldr = next;
+ next = temp;
+ next.Length = 0;
+ minIx = brkIx;
+ brkIx = maxIx;
+ }
+ }
+ }
+
+ BufferElement data = new BufferElement();
+
+ int bPos; // Postion index in the StringBuilder
+ BlockReader NextBlk; // Delegate that serves char-arrays;
+
+ private string EncodingName
+ {
+ get
+ {
+ StreamReader rdr = NextBlk.Target as StreamReader;
+ return (rdr == null ? "raw-bytes" : rdr.CurrentEncoding.BodyName);
+ }
+ }
+
+ public BuildBuffer(Stream stream)
+ {
+ FileStream fStrm = (stream as FileStream);
+ if (fStrm != null) FileName = fStrm.Name;
+ NextBlk = BlockReaderFactory.Raw(stream);
+ }
+
+#if (!BYTEMODE)
+ public BuildBuffer(Stream stream, int fallbackCodePage)
+ {
+ FileStream fStrm = (stream as FileStream);
+ if (fStrm != null) FileName = fStrm.Name;
+ NextBlk = BlockReaderFactory.Get(stream, fallbackCodePage);
+ }
+#endif
+
+ ///
+ /// Marks a conservative lower bound for the buffer,
+ /// allowing space to be reclaimed. If an application
+ /// needs to call GetString at arbitrary past locations
+ /// in the input stream, Mark() is not called.
+ ///
+ public override void Mark() { data.Mark(bPos - 2); }
+
+ public override int Pos
+ {
+ get { return bPos; }
+ set { bPos = value; }
+ }
+
+
+ ///
+ /// Read returns the ordinal number of the next char, or
+ /// EOF (-1) for an end of stream. Note that the next
+ /// code point may require *two* calls of Read().
+ ///
+ ///
+ public override int Read()
+ {
+ //
+ // Characters at positions
+ // [data.offset, data.offset + data.bldr.Length)
+ // are available in data.bldr.
+ //
+ if (bPos < data.MaxIndex)
+ {
+ // ch0 cannot be EOF
+ return (int)data[bPos++];
+ }
+ else // Read from underlying stream
+ {
+ // Experimental code, blocks of page size
+ char[] chrs = new char[4096];
+ int count = NextBlk(chrs, 0, 4096);
+ if (count == 0)
+ return EndOfFile;
+ else
+ {
+ data.Append(chrs, count);
+ return (int)data[bPos++];
+ }
+ }
+ }
+
+ public override string GetString(int begin, int limit)
+ {
+ return data.GetString(begin, limit);
+ }
+
+ public override string ToString()
+ {
+ return "StringBuilder buffer, encoding: " + this.EncodingName;
+ }
+ }
+
+ // =============== End ScanBuff-derived classes ==================
+
+ internal delegate int BlockReader(char[] block, int index, int number);
+
+ // A delegate factory, serving up a delegate that
+ // reads a block of characters from the underlying
+ // encoded stream, via a StreamReader object.
+ //
+ internal static class BlockReaderFactory
+ {
+ public static BlockReader Raw(Stream stream)
+ {
+ return delegate(char[] block, int index, int number)
+ {
+ byte[] b = new byte[number];
+ int count = stream.Read(b, 0, number);
+ int i = 0;
+ int j = index;
+ for (; i < count; i++, j++)
+ block[j] = (char)b[i];
+ return count;
+ };
+ }
+
+#if (!BYTEMODE)
+ public static BlockReader Get(Stream stream, int fallbackCodePage)
+ {
+ Encoding encoding;
+ int preamble = Preamble(stream);
+
+ if (preamble != 0) // There is a valid BOM here!
+ encoding = Encoding.GetEncoding(preamble);
+ else if (fallbackCodePage == -1) // Fallback is "raw" bytes
+ return Raw(stream);
+ else if (fallbackCodePage != -2) // Anything but "guess"
+ encoding = Encoding.GetEncoding(fallbackCodePage);
+ else // This is the "guess" option
+ {
+ int guess = new Guesser(stream).GuessCodePage();
+ stream.Seek(0, SeekOrigin.Begin);
+ if (guess == -1) // ==> this is a 7-bit file
+ encoding = Encoding.ASCII;
+ else if (guess == 65001)
+ encoding = Encoding.UTF8;
+ else // ==> use the machine default
+ encoding = Encoding.Default;
+ }
+ StreamReader reader = new StreamReader(stream, encoding);
+ return reader.Read;
+ }
+
+ static int Preamble(Stream stream)
+ {
+ int b0 = stream.ReadByte();
+ int b1 = stream.ReadByte();
+
+ if (b0 == 0xfe && b1 == 0xff)
+ return 1201; // UTF16BE
+ if (b0 == 0xff && b1 == 0xfe)
+ return 1200; // UTF16LE
+
+ int b2 = stream.ReadByte();
+ if (b0 == 0xef && b1 == 0xbb && b2 == 0xbf)
+ return 65001; // UTF8
+ //
+ // There is no unicode preamble, so we
+ // return denoter for the machine default.
+ //
+ stream.Seek(0, SeekOrigin.Begin);
+ return 0;
+ }
+#endif // !BYTEMODE
+ }
+#endif // !NOFILES
+ #endregion Buffer classes
+
+ // ==============================================================
+ // ============ class CodePageHandling =============
+ // ==============================================================
+#if (!NOFILES)
+ public static class CodePageHandling
+ {
+ public static int GetCodePage(string option)
+ {
+ string command = option.ToUpperInvariant();
+ if (command.StartsWith("CodePage:", StringComparison.OrdinalIgnoreCase))
+ command = command.Substring(9);
+ try
+ {
+ if (command.Equals("RAW"))
+ return -1;
+ else if (command.Equals("GUESS"))
+ return -2;
+ else if (command.Equals("DEFAULT"))
+ return 0;
+ else if (char.IsDigit(command[0]))
+ return int.Parse(command, CultureInfo.InvariantCulture);
+ else
+ {
+ Encoding enc = Encoding.GetEncoding(command);
+ return enc.CodePage;
+ }
+ }
+ catch (FormatException)
+ {
+ Console.Error.WriteLine(
+ "Invalid format \"{0}\", using machine default", option);
+ }
+ catch (ArgumentException)
+ {
+ Console.Error.WriteLine(
+ "Unknown code page \"{0}\", using machine default", option);
+ }
+ return 0;
+ }
+ }
+#region guesser
+#if (!BYTEMODE)
+ // ==============================================================
+ // ============ Encoding Guesser =============
+ // ==============================================================
+
+ ///
+ /// This class provides a simple finite state automaton that
+ /// scans the file looking for (1) valid UTF-8 byte patterns,
+ /// (2) bytes >= 0x80 which are not part of a UTF-8 sequence.
+ /// The method then guesses whether it is UTF-8 or maybe some
+ /// local machine default encoding. This works well for the
+ /// various Latin encodings.
+ ///
+ internal class Guesser
+ {
+ ScanBuff buffer;
+
+ public int GuessCodePage() { return Scan(); }
+
+ const int maxAccept = 10;
+ const int initial = 0;
+ const int eofNum = 0;
+ const int goStart = -1;
+ const int INITIAL = 0;
+ const int EndToken = 0;
+
+ #region user code
+ /*
+ * Reads the bytes of a file to determine if it is
+ * UTF-8 or a single-byte code page file.
+ */
+ public long utfX;
+ public long uppr;
+ #endregion user code
+
+ int state;
+ int currentStart = startState[0];
+ int code;
+
+ #region ScannerTables
+ static int[] startState = new int[] { 11, 0 };
+
+ #region CharacterMap
+ static sbyte[] map = new sbyte[256] {
+/* '\0' */ 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0,
+/* '\x10' */ 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0,
+/* '\x20' */ 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0,
+/* '0' */ 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0,
+/* '@' */ 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0,
+/* 'P' */ 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0,
+/* '`' */ 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0,
+/* 'p' */ 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0,
+/* '\x80' */ 2, 2, 2, 2, 2, 2, 2, 2, 2, 2, 2, 2, 2, 2, 2, 2,
+/* '\x90' */ 2, 2, 2, 2, 2, 2, 2, 2, 2, 2, 2, 2, 2, 2, 2, 2,
+/* '\xA0' */ 2, 2, 2, 2, 2, 2, 2, 2, 2, 2, 2, 2, 2, 2, 2, 2,
+/* '\xB0' */ 2, 2, 2, 2, 2, 2, 2, 2, 2, 2, 2, 2, 2, 2, 2, 2,
+/* '\xC0' */ 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1,
+/* '\xD0' */ 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1,
+/* '\xE0' */ 3, 3, 3, 3, 3, 3, 3, 3, 3, 3, 3, 3, 3, 3, 3, 3,
+/* '\xF0' */ 4, 4, 4, 4, 4, 4, 4, 4, 5, 5, 5, 5, 5, 5, 5, 5 };
+ #endregion
+
+ static sbyte[][] nextState = new sbyte[][] {
+ new sbyte[] {0, 0, 0, 0, 0, 0},
+ new sbyte[] {-1, -1, 10, -1, -1, -1},
+ new sbyte[] {-1, -1, -1, -1, -1, -1},
+ new sbyte[] {-1, -1, 8, -1, -1, -1},
+ new sbyte[] {-1, -1, 5, -1, -1, -1},
+ new sbyte[] {-1, -1, 6, -1, -1, -1},
+ new sbyte[] {-1, -1, 7, -1, -1, -1},
+ null,
+ new sbyte[] {-1, -1, 9, -1, -1, -1},
+ null,
+ null,
+ new sbyte[] {-1, 1, 2, 3, 4, 2}
+ };
+
+
+ [SuppressMessage("Microsoft.Performance", "CA1810:InitializeReferenceTypeStaticFieldsInline")]
+ // Reason for suppression: cannot have self-reference in array initializer.
+ static Guesser()
+ {
+ nextState[7] = nextState[2];
+ nextState[9] = nextState[2];
+ nextState[10] = nextState[2];
+ }
+
+ int NextState()
+ {
+ if (code == ScanBuff.EndOfFile)
+ return eofNum;
+ else
+ return nextState[state][map[code]];
+ }
+ #endregion
+
+ public Guesser(System.IO.Stream file) { SetSource(file); }
+
+ public void SetSource(System.IO.Stream source)
+ {
+ this.buffer = new BuildBuffer(source);
+ code = buffer.Read();
+ }
+
+ int Scan()
+ {
+ for (; ; )
+ {
+ int next;
+ state = currentStart;
+ while ((next = NextState()) == goStart)
+ code = buffer.Read();
+
+ state = next;
+ code = buffer.Read();
+
+ while ((next = NextState()) > eofNum)
+ {
+ state = next;
+ code = buffer.Read();
+ }
+ if (state <= maxAccept)
+ {
+ #region ActionSwitch
+#pragma warning disable 162
+ switch (state)
+ {
+ case eofNum:
+ switch (currentStart)
+ {
+ case 11:
+ if (utfX == 0 && uppr == 0) return -1; /* raw ascii */
+ else if (uppr * 10 > utfX) return 0; /* default code page */
+ else return 65001; /* UTF-8 encoding */
+ break;
+ }
+ return EndToken;
+ case 1: // Recognized '{Upper128}', Shortest string "\xC0"
+ case 2: // Recognized '{Upper128}', Shortest string "\x80"
+ case 3: // Recognized '{Upper128}', Shortest string "\xE0"
+ case 4: // Recognized '{Upper128}', Shortest string "\xF0"
+ uppr++;
+ break;
+ case 5: // Recognized '{Utf8pfx4}{Utf8cont}', Shortest string "\xF0\x80"
+ uppr += 2;
+ break;
+ case 6: // Recognized '{Utf8pfx4}{Utf8cont}{2}', Shortest string "\xF0\x80\x80"
+ uppr += 3;
+ break;
+ case 7: // Recognized '{Utf8pfx4}{Utf8cont}{3}', Shortest string "\xF0\x80\x80\x80"
+ utfX += 3;
+ break;
+ case 8: // Recognized '{Utf8pfx3}{Utf8cont}', Shortest string "\xE0\x80"
+ uppr += 2;
+ break;
+ case 9: // Recognized '{Utf8pfx3}{Utf8cont}{2}', Shortest string "\xE0\x80\x80"
+ utfX += 2;
+ break;
+ case 10: // Recognized '{Utf8pfx2}{Utf8cont}', Shortest string "\xC0\x80"
+ utfX++;
+ break;
+ default:
+ break;
+ }
+#pragma warning restore 162
+ #endregion
+ }
+ }
+ }
+ } // end class Guesser
+
+#endif // !BYTEMODE
+#endregion
+#endif // !NOFILES
+
+// End of code copied from embedded resource
+}
diff --git a/src/DataStreamSuite/DataStreamPublisher/Program.cs b/src/DataStreamSuite/DataStreamPublisher/Program.cs
index f0eb52994..a86fd2e2a 100644
--- a/src/DataStreamSuite/DataStreamPublisher/Program.cs
+++ b/src/DataStreamSuite/DataStreamPublisher/Program.cs
@@ -70,6 +70,7 @@ void _publish_streams() {
//{"books","books()"},
{"seconds_1_10","second(1,10)"},
{"literal_420","literal(\"x\", \"420.0\")"},
+ {"tradebar_30","tradebars(30)"},
{"mongodb_testcollection", "mongodb(\"mongodb://192.168.1.80:27017\",\"local\",\"test_collection\")"}
};
// {"bbo_ts","timeSeries(bbo())"}