diff --git a/src/DataStreamSuite/ArithmeticExpressions/GplexBuffers.cs b/src/DataStreamSuite/ArithmeticExpressions/GplexBuffers.cs index 76dd35443..1bcc08e09 100644 --- a/src/DataStreamSuite/ArithmeticExpressions/GplexBuffers.cs +++ b/src/DataStreamSuite/ArithmeticExpressions/GplexBuffers.cs @@ -1,720 +1,720 @@ -// ============================================================== -// -// This code automatically produced from an embedded resource. -// Do not edit this file, or it will become incompatible with -// the specification from which it was generated. -// -// ============================================================== - -using System; -using System.IO; -using System.Text; -using System.Collections.Generic; -using System.Diagnostics.CodeAnalysis; -using System.Runtime.Serialization; -using System.Globalization; - -namespace QUT.GplexBuffers -{ -// Code copied from GPLEX embedded resource - [Serializable] - internal class BufferException : Exception - { - public BufferException() { } - public BufferException(string message) : base(message) { } - public BufferException(string message, Exception innerException) - : base(message, innerException) { } - protected BufferException(SerializationInfo info, StreamingContext context) - : base(info, context) { } - } - - internal abstract class ScanBuff - { - private string fileNm; - - public const int EndOfFile = -1; - public const int UnicodeReplacementChar = 0xFFFD; - - public bool IsFile { get { return (fileNm != null); } } - public string FileName { get { return fileNm; } set { fileNm = value; } } - - public abstract int Pos { get; set; } - public abstract int Read(); - public virtual void Mark() { } - - public abstract string GetString(int begin, int limit); - - public static ScanBuff GetBuffer(string source) - { - return new StringBuffer(source); - } - - public static ScanBuff GetBuffer(IList source) - { - return new LineBuffer(source); - } - -#if (!NOFILES) - public static ScanBuff GetBuffer(Stream source) - { - return new BuildBuffer(source); - } - -#if (!BYTEMODE) - public static ScanBuff GetBuffer(Stream source, int fallbackCodePage) - { - return new BuildBuffer(source, fallbackCodePage); - } -#endif // !BYTEMODE -#endif // !NOFILES - } - - #region Buffer classes - - // ============================================================== - // ===== Definitions for various ScanBuff derived classes ==== - // ============================================================== - // =============== String input ================ - // ============================================================== - - /// - /// This class reads characters from a single string as - /// required, for example, by Visual Studio language services - /// - sealed class StringBuffer : ScanBuff - { - string str; // input buffer - int bPos; // current position in buffer - int sLen; - - public StringBuffer(string source) - { - this.str = source; - this.sLen = source.Length; - this.FileName = null; - } - - public override int Read() - { - if (bPos < sLen) return str[bPos++]; - else if (bPos == sLen) { bPos++; return '\n'; } // one strike, see new line - else { bPos++; return EndOfFile; } // two strikes and you're out! - } - - public override string GetString(int begin, int limit) - { - // "limit" can be greater than sLen with the BABEL - // option set. Read returns a "virtual" EOL if - // an attempt is made to read past the end of the - // string buffer. Without the guard any attempt - // to fetch yytext for a token that includes the - // EOL will throw an index exception. - if (limit > sLen) limit = sLen; - if (limit <= begin) return ""; - else return str.Substring(begin, limit - begin); - } - - public override int Pos - { - get { return bPos; } - set { bPos = value; } - } - - public override string ToString() { return "StringBuffer"; } - } - - // ============================================================== - // The LineBuff class contributed by Nigel Horspool, - // nigelh@cs.uvic.cs - // ============================================================== - - sealed class LineBuffer : ScanBuff - { - IList line; // list of source lines from a file - int numLines; // number of strings in line list - string curLine; // current line in that list - int cLine; // index of current line in the list - int curLen; // length of current line - int curLineStart; // position of line start in whole file - int curLineEnd; // position of line end in whole file - int maxPos; // max position ever visited in whole file - int cPos; // ordinal number of code in source - - // Constructed from a list of strings, one per source line. - // The lines have had trailing '\n' characters removed. - public LineBuffer(IList lineList) - { - line = lineList; - numLines = line.Count; - cPos = curLineStart = 0; - curLine = (numLines > 0 ? line[0] : ""); - maxPos = curLineEnd = curLen = curLine.Length; - cLine = 1; - FileName = null; - } - - public override int Read() - { - if (cPos < curLineEnd) - return curLine[cPos++ - curLineStart]; - if (cPos++ == curLineEnd) - return '\n'; - if (cLine >= numLines) - return EndOfFile; - curLine = line[cLine]; - curLen = curLine.Length; - curLineStart = curLineEnd + 1; - curLineEnd = curLineStart + curLen; - if (curLineEnd > maxPos) - maxPos = curLineEnd; - cLine++; - return curLen > 0 ? curLine[0] : '\n'; - } - - // To speed up searches for the line containing a position - private int cachedPosition; - private int cachedIxdex; - private int cachedLineStart; - - // Given a position pos within the entire source, the results are - // ix -- the index of the containing line - // lstart -- the position of the first character on that line - private void findIndex(int pos, out int ix, out int lstart) - { - if (pos >= cachedPosition) - { - ix = cachedIxdex; lstart = cachedLineStart; - } - else - { - ix = lstart = 0; - } - while (ix < numLines) - { - int len = line[ix].Length + 1; - if (pos < lstart + len) break; - lstart += len; - ix++; - } - cachedPosition = pos; - cachedIxdex = ix; - cachedLineStart = lstart; - } - - public override string GetString(int begin, int limit) - { - if (begin >= maxPos || limit <= begin) return ""; - int endIx, begIx, endLineStart, begLineStart; - findIndex(begin, out begIx, out begLineStart); - int begCol = begin - begLineStart; - findIndex(limit, out endIx, out endLineStart); - int endCol = limit - endLineStart; - string s = line[begIx]; - if (begIx == endIx) - { - // the usual case, substring all on one line - return (endCol <= s.Length) ? - s.Substring(begCol, endCol - begCol) - : s.Substring(begCol) + "\n"; - } - // the string spans multiple lines, yuk! - StringBuilder sb = new StringBuilder(); - if (begCol < s.Length) - sb.Append(s.Substring(begCol)); - for (; ; ) - { - sb.Append("\n"); - s = line[++begIx]; - if (begIx >= endIx) break; - sb.Append(s); - } - if (endCol <= s.Length) - { - sb.Append(s.Substring(0, endCol)); - } - else - { - sb.Append(s); - sb.Append("\n"); - } - return sb.ToString(); - } - - public override int Pos - { - get { return cPos; } - set - { - cPos = value; - findIndex(cPos, out cLine, out curLineStart); - // cLine should be the *next* line after curLine. - curLine = (cLine < numLines ? line[cLine++] : ""); - curLineEnd = curLineStart + curLine.Length; - } - } - - public override string ToString() { return "LineBuffer"; } - } - -#if (!NOFILES) - // ============================================================== - // ===== class BuildBuff : for unicode text files ======== - // ============================================================== - - class BuildBuffer : ScanBuff - { - // Double buffer for char stream. - class BufferElement - { - StringBuilder bldr = new StringBuilder(); - StringBuilder next = new StringBuilder(); - int minIx; - int maxIx; - int brkIx; - bool appendToNext; - - internal BufferElement() { } - - internal int MaxIndex { get { return maxIx; } } - // internal int MinIndex { get { return minIx; } } - - internal char this[int index] - { - get - { - if (index < minIx || index >= maxIx) - throw new BufferException("Index was outside data buffer"); - else if (index < brkIx) - return bldr[index - minIx]; - else - return next[index - brkIx]; - } - } - - internal void Append(char[] block, int count) - { - maxIx += count; - if (appendToNext) - this.next.Append(block, 0, count); - else - { - this.bldr.Append(block, 0, count); - brkIx = maxIx; - appendToNext = true; - } - } - - internal string GetString(int start, int limit) - { - if (limit <= start) - return ""; - if (start >= minIx && limit <= maxIx) - if (limit < brkIx) // String entirely in bldr builder - return bldr.ToString(start - minIx, limit - start); - else if (start >= brkIx) // String entirely in next builder - return next.ToString(start - brkIx, limit - start); - else // Must do a string-concatenation - return - bldr.ToString(start - minIx, brkIx - start) + - next.ToString(0, limit - brkIx); - else - throw new BufferException("String was outside data buffer"); - } - - internal void Mark(int limit) - { - if (limit > brkIx + 16) // Rotate blocks - { - StringBuilder temp = bldr; - bldr = next; - next = temp; - next.Length = 0; - minIx = brkIx; - brkIx = maxIx; - } - } - } - - BufferElement data = new BufferElement(); - - int bPos; // Postion index in the StringBuilder - BlockReader NextBlk; // Delegate that serves char-arrays; - - private string EncodingName - { - get - { - StreamReader rdr = NextBlk.Target as StreamReader; - return (rdr == null ? "raw-bytes" : rdr.CurrentEncoding.BodyName); - } - } - - public BuildBuffer(Stream stream) - { - FileStream fStrm = (stream as FileStream); - if (fStrm != null) FileName = fStrm.Name; - NextBlk = BlockReaderFactory.Raw(stream); - } - -#if (!BYTEMODE) - public BuildBuffer(Stream stream, int fallbackCodePage) - { - FileStream fStrm = (stream as FileStream); - if (fStrm != null) FileName = fStrm.Name; - NextBlk = BlockReaderFactory.Get(stream, fallbackCodePage); - } -#endif - - /// - /// Marks a conservative lower bound for the buffer, - /// allowing space to be reclaimed. If an application - /// needs to call GetString at arbitrary past locations - /// in the input stream, Mark() is not called. - /// - public override void Mark() { data.Mark(bPos - 2); } - - public override int Pos - { - get { return bPos; } - set { bPos = value; } - } - - - /// - /// Read returns the ordinal number of the next char, or - /// EOF (-1) for an end of stream. Note that the next - /// code point may require *two* calls of Read(). - /// - /// - public override int Read() - { - // - // Characters at positions - // [data.offset, data.offset + data.bldr.Length) - // are available in data.bldr. - // - if (bPos < data.MaxIndex) - { - // ch0 cannot be EOF - return (int)data[bPos++]; - } - else // Read from underlying stream - { - // Experimental code, blocks of page size - char[] chrs = new char[4096]; - int count = NextBlk(chrs, 0, 4096); - if (count == 0) - return EndOfFile; - else - { - data.Append(chrs, count); - return (int)data[bPos++]; - } - } - } - - public override string GetString(int begin, int limit) - { - return data.GetString(begin, limit); - } - - public override string ToString() - { - return "StringBuilder buffer, encoding: " + this.EncodingName; - } - } - - // =============== End ScanBuff-derived classes ================== - - internal delegate int BlockReader(char[] block, int index, int number); - - // A delegate factory, serving up a delegate that - // reads a block of characters from the underlying - // encoded stream, via a StreamReader object. - // - internal static class BlockReaderFactory - { - public static BlockReader Raw(Stream stream) - { - return delegate(char[] block, int index, int number) - { - byte[] b = new byte[number]; - int count = stream.Read(b, 0, number); - int i = 0; - int j = index; - for (; i < count; i++, j++) - block[j] = (char)b[i]; - return count; - }; - } - -#if (!BYTEMODE) - public static BlockReader Get(Stream stream, int fallbackCodePage) - { - Encoding encoding; - int preamble = Preamble(stream); - - if (preamble != 0) // There is a valid BOM here! - encoding = Encoding.GetEncoding(preamble); - else if (fallbackCodePage == -1) // Fallback is "raw" bytes - return Raw(stream); - else if (fallbackCodePage != -2) // Anything but "guess" - encoding = Encoding.GetEncoding(fallbackCodePage); - else // This is the "guess" option - { - int guess = new Guesser(stream).GuessCodePage(); - stream.Seek(0, SeekOrigin.Begin); - if (guess == -1) // ==> this is a 7-bit file - encoding = Encoding.ASCII; - else if (guess == 65001) - encoding = Encoding.UTF8; - else // ==> use the machine default - encoding = Encoding.Default; - } - StreamReader reader = new StreamReader(stream, encoding); - return reader.Read; - } - - static int Preamble(Stream stream) - { - int b0 = stream.ReadByte(); - int b1 = stream.ReadByte(); - - if (b0 == 0xfe && b1 == 0xff) - return 1201; // UTF16BE - if (b0 == 0xff && b1 == 0xfe) - return 1200; // UTF16LE - - int b2 = stream.ReadByte(); - if (b0 == 0xef && b1 == 0xbb && b2 == 0xbf) - return 65001; // UTF8 - // - // There is no unicode preamble, so we - // return denoter for the machine default. - // - stream.Seek(0, SeekOrigin.Begin); - return 0; - } -#endif // !BYTEMODE - } -#endif // !NOFILES - #endregion Buffer classes - - // ============================================================== - // ============ class CodePageHandling ============= - // ============================================================== -#if (!NOFILES) - public static class CodePageHandling - { - public static int GetCodePage(string option) - { - string command = option.ToUpperInvariant(); - if (command.StartsWith("CodePage:", StringComparison.OrdinalIgnoreCase)) - command = command.Substring(9); - try - { - if (command.Equals("RAW")) - return -1; - else if (command.Equals("GUESS")) - return -2; - else if (command.Equals("DEFAULT")) - return 0; - else if (char.IsDigit(command[0])) - return int.Parse(command, CultureInfo.InvariantCulture); - else - { - Encoding enc = Encoding.GetEncoding(command); - return enc.CodePage; - } - } - catch (FormatException) - { - Console.Error.WriteLine( - "Invalid format \"{0}\", using machine default", option); - } - catch (ArgumentException) - { - Console.Error.WriteLine( - "Unknown code page \"{0}\", using machine default", option); - } - return 0; - } - } -#region guesser -#if (!BYTEMODE) - // ============================================================== - // ============ Encoding Guesser ============= - // ============================================================== - - /// - /// This class provides a simple finite state automaton that - /// scans the file looking for (1) valid UTF-8 byte patterns, - /// (2) bytes >= 0x80 which are not part of a UTF-8 sequence. - /// The method then guesses whether it is UTF-8 or maybe some - /// local machine default encoding. This works well for the - /// various Latin encodings. - /// - internal class Guesser - { - ScanBuff buffer; - - public int GuessCodePage() { return Scan(); } - - const int maxAccept = 10; - const int initial = 0; - const int eofNum = 0; - const int goStart = -1; - const int INITIAL = 0; - const int EndToken = 0; - - #region user code - /* - * Reads the bytes of a file to determine if it is - * UTF-8 or a single-byte code page file. - */ - public long utfX; - public long uppr; - #endregion user code - - int state; - int currentStart = startState[0]; - int code; - - #region ScannerTables - static int[] startState = new int[] { 11, 0 }; - - #region CharacterMap - static sbyte[] map = new sbyte[256] { -/* '\0' */ 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, -/* '\x10' */ 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, -/* '\x20' */ 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, -/* '0' */ 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, -/* '@' */ 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, -/* 'P' */ 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, -/* '`' */ 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, -/* 'p' */ 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, -/* '\x80' */ 2, 2, 2, 2, 2, 2, 2, 2, 2, 2, 2, 2, 2, 2, 2, 2, -/* '\x90' */ 2, 2, 2, 2, 2, 2, 2, 2, 2, 2, 2, 2, 2, 2, 2, 2, -/* '\xA0' */ 2, 2, 2, 2, 2, 2, 2, 2, 2, 2, 2, 2, 2, 2, 2, 2, -/* '\xB0' */ 2, 2, 2, 2, 2, 2, 2, 2, 2, 2, 2, 2, 2, 2, 2, 2, -/* '\xC0' */ 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, -/* '\xD0' */ 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, -/* '\xE0' */ 3, 3, 3, 3, 3, 3, 3, 3, 3, 3, 3, 3, 3, 3, 3, 3, -/* '\xF0' */ 4, 4, 4, 4, 4, 4, 4, 4, 5, 5, 5, 5, 5, 5, 5, 5 }; - #endregion - - static sbyte[][] nextState = new sbyte[][] { - new sbyte[] {0, 0, 0, 0, 0, 0}, - new sbyte[] {-1, -1, 10, -1, -1, -1}, - new sbyte[] {-1, -1, -1, -1, -1, -1}, - new sbyte[] {-1, -1, 8, -1, -1, -1}, - new sbyte[] {-1, -1, 5, -1, -1, -1}, - new sbyte[] {-1, -1, 6, -1, -1, -1}, - new sbyte[] {-1, -1, 7, -1, -1, -1}, - null, - new sbyte[] {-1, -1, 9, -1, -1, -1}, - null, - null, - new sbyte[] {-1, 1, 2, 3, 4, 2} - }; - - - [SuppressMessage("Microsoft.Performance", "CA1810:InitializeReferenceTypeStaticFieldsInline")] - // Reason for suppression: cannot have self-reference in array initializer. - static Guesser() - { - nextState[7] = nextState[2]; - nextState[9] = nextState[2]; - nextState[10] = nextState[2]; - } - - int NextState() - { - if (code == ScanBuff.EndOfFile) - return eofNum; - else - return nextState[state][map[code]]; - } - #endregion - - public Guesser(System.IO.Stream file) { SetSource(file); } - - public void SetSource(System.IO.Stream source) - { - this.buffer = new BuildBuffer(source); - code = buffer.Read(); - } - - int Scan() - { - for (; ; ) - { - int next; - state = currentStart; - while ((next = NextState()) == goStart) - code = buffer.Read(); - - state = next; - code = buffer.Read(); - - while ((next = NextState()) > eofNum) - { - state = next; - code = buffer.Read(); - } - if (state <= maxAccept) - { - #region ActionSwitch -#pragma warning disable 162 - switch (state) - { - case eofNum: - switch (currentStart) - { - case 11: - if (utfX == 0 && uppr == 0) return -1; /* raw ascii */ - else if (uppr * 10 > utfX) return 0; /* default code page */ - else return 65001; /* UTF-8 encoding */ - break; - } - return EndToken; - case 1: // Recognized '{Upper128}', Shortest string "\xC0" - case 2: // Recognized '{Upper128}', Shortest string "\x80" - case 3: // Recognized '{Upper128}', Shortest string "\xE0" - case 4: // Recognized '{Upper128}', Shortest string "\xF0" - uppr++; - break; - case 5: // Recognized '{Utf8pfx4}{Utf8cont}', Shortest string "\xF0\x80" - uppr += 2; - break; - case 6: // Recognized '{Utf8pfx4}{Utf8cont}{2}', Shortest string "\xF0\x80\x80" - uppr += 3; - break; - case 7: // Recognized '{Utf8pfx4}{Utf8cont}{3}', Shortest string "\xF0\x80\x80\x80" - utfX += 3; - break; - case 8: // Recognized '{Utf8pfx3}{Utf8cont}', Shortest string "\xE0\x80" - uppr += 2; - break; - case 9: // Recognized '{Utf8pfx3}{Utf8cont}{2}', Shortest string "\xE0\x80\x80" - utfX += 2; - break; - case 10: // Recognized '{Utf8pfx2}{Utf8cont}', Shortest string "\xC0\x80" - utfX++; - break; - default: - break; - } -#pragma warning restore 162 - #endregion - } - } - } - } // end class Guesser - -#endif // !BYTEMODE -#endregion -#endif // !NOFILES - -// End of code copied from embedded resource -} +// ============================================================== +// +// This code automatically produced from an embedded resource. +// Do not edit this file, or it will become incompatible with +// the specification from which it was generated. +// +// ============================================================== + +using System; +using System.IO; +using System.Text; +using System.Collections.Generic; +using System.Diagnostics.CodeAnalysis; +using System.Runtime.Serialization; +using System.Globalization; + +namespace QUT.GplexBuffers +{ +// Code copied from GPLEX embedded resource + [Serializable] + internal class BufferException : Exception + { + public BufferException() { } + public BufferException(string message) : base(message) { } + public BufferException(string message, Exception innerException) + : base(message, innerException) { } + protected BufferException(SerializationInfo info, StreamingContext context) + : base(info, context) { } + } + + internal abstract class ScanBuff + { + private string fileNm; + + public const int EndOfFile = -1; + public const int UnicodeReplacementChar = 0xFFFD; + + public bool IsFile { get { return (fileNm != null); } } + public string FileName { get { return fileNm; } set { fileNm = value; } } + + public abstract int Pos { get; set; } + public abstract int Read(); + public virtual void Mark() { } + + public abstract string GetString(int begin, int limit); + + public static ScanBuff GetBuffer(string source) + { + return new StringBuffer(source); + } + + public static ScanBuff GetBuffer(IList source) + { + return new LineBuffer(source); + } + +#if (!NOFILES) + public static ScanBuff GetBuffer(Stream source) + { + return new BuildBuffer(source); + } + +#if (!BYTEMODE) + public static ScanBuff GetBuffer(Stream source, int fallbackCodePage) + { + return new BuildBuffer(source, fallbackCodePage); + } +#endif // !BYTEMODE +#endif // !NOFILES + } + + #region Buffer classes + + // ============================================================== + // ===== Definitions for various ScanBuff derived classes ==== + // ============================================================== + // =============== String input ================ + // ============================================================== + + /// + /// This class reads characters from a single string as + /// required, for example, by Visual Studio language services + /// + sealed class StringBuffer : ScanBuff + { + string str; // input buffer + int bPos; // current position in buffer + int sLen; + + public StringBuffer(string source) + { + this.str = source; + this.sLen = source.Length; + this.FileName = null; + } + + public override int Read() + { + if (bPos < sLen) return str[bPos++]; + else if (bPos == sLen) { bPos++; return '\n'; } // one strike, see new line + else { bPos++; return EndOfFile; } // two strikes and you're out! + } + + public override string GetString(int begin, int limit) + { + // "limit" can be greater than sLen with the BABEL + // option set. Read returns a "virtual" EOL if + // an attempt is made to read past the end of the + // string buffer. Without the guard any attempt + // to fetch yytext for a token that includes the + // EOL will throw an index exception. + if (limit > sLen) limit = sLen; + if (limit <= begin) return ""; + else return str.Substring(begin, limit - begin); + } + + public override int Pos + { + get { return bPos; } + set { bPos = value; } + } + + public override string ToString() { return "StringBuffer"; } + } + + // ============================================================== + // The LineBuff class contributed by Nigel Horspool, + // nigelh@cs.uvic.cs + // ============================================================== + + sealed class LineBuffer : ScanBuff + { + IList line; // list of source lines from a file + int numLines; // number of strings in line list + string curLine; // current line in that list + int cLine; // index of current line in the list + int curLen; // length of current line + int curLineStart; // position of line start in whole file + int curLineEnd; // position of line end in whole file + int maxPos; // max position ever visited in whole file + int cPos; // ordinal number of code in source + + // Constructed from a list of strings, one per source line. + // The lines have had trailing '\n' characters removed. + public LineBuffer(IList lineList) + { + line = lineList; + numLines = line.Count; + cPos = curLineStart = 0; + curLine = (numLines > 0 ? line[0] : ""); + maxPos = curLineEnd = curLen = curLine.Length; + cLine = 1; + FileName = null; + } + + public override int Read() + { + if (cPos < curLineEnd) + return curLine[cPos++ - curLineStart]; + if (cPos++ == curLineEnd) + return '\n'; + if (cLine >= numLines) + return EndOfFile; + curLine = line[cLine]; + curLen = curLine.Length; + curLineStart = curLineEnd + 1; + curLineEnd = curLineStart + curLen; + if (curLineEnd > maxPos) + maxPos = curLineEnd; + cLine++; + return curLen > 0 ? curLine[0] : '\n'; + } + + // To speed up searches for the line containing a position + private int cachedPosition; + private int cachedIxdex; + private int cachedLineStart; + + // Given a position pos within the entire source, the results are + // ix -- the index of the containing line + // lstart -- the position of the first character on that line + private void findIndex(int pos, out int ix, out int lstart) + { + if (pos >= cachedPosition) + { + ix = cachedIxdex; lstart = cachedLineStart; + } + else + { + ix = lstart = 0; + } + while (ix < numLines) + { + int len = line[ix].Length + 1; + if (pos < lstart + len) break; + lstart += len; + ix++; + } + cachedPosition = pos; + cachedIxdex = ix; + cachedLineStart = lstart; + } + + public override string GetString(int begin, int limit) + { + if (begin >= maxPos || limit <= begin) return ""; + int endIx, begIx, endLineStart, begLineStart; + findIndex(begin, out begIx, out begLineStart); + int begCol = begin - begLineStart; + findIndex(limit, out endIx, out endLineStart); + int endCol = limit - endLineStart; + string s = line[begIx]; + if (begIx == endIx) + { + // the usual case, substring all on one line + return (endCol <= s.Length) ? + s.Substring(begCol, endCol - begCol) + : s.Substring(begCol) + "\n"; + } + // the string spans multiple lines, yuk! + StringBuilder sb = new StringBuilder(); + if (begCol < s.Length) + sb.Append(s.Substring(begCol)); + for (; ; ) + { + sb.Append("\n"); + s = line[++begIx]; + if (begIx >= endIx) break; + sb.Append(s); + } + if (endCol <= s.Length) + { + sb.Append(s.Substring(0, endCol)); + } + else + { + sb.Append(s); + sb.Append("\n"); + } + return sb.ToString(); + } + + public override int Pos + { + get { return cPos; } + set + { + cPos = value; + findIndex(cPos, out cLine, out curLineStart); + // cLine should be the *next* line after curLine. + curLine = (cLine < numLines ? line[cLine++] : ""); + curLineEnd = curLineStart + curLine.Length; + } + } + + public override string ToString() { return "LineBuffer"; } + } + +#if (!NOFILES) + // ============================================================== + // ===== class BuildBuff : for unicode text files ======== + // ============================================================== + + class BuildBuffer : ScanBuff + { + // Double buffer for char stream. + class BufferElement + { + StringBuilder bldr = new StringBuilder(); + StringBuilder next = new StringBuilder(); + int minIx; + int maxIx; + int brkIx; + bool appendToNext; + + internal BufferElement() { } + + internal int MaxIndex { get { return maxIx; } } + // internal int MinIndex { get { return minIx; } } + + internal char this[int index] + { + get + { + if (index < minIx || index >= maxIx) + throw new BufferException("Index was outside data buffer"); + else if (index < brkIx) + return bldr[index - minIx]; + else + return next[index - brkIx]; + } + } + + internal void Append(char[] block, int count) + { + maxIx += count; + if (appendToNext) + this.next.Append(block, 0, count); + else + { + this.bldr.Append(block, 0, count); + brkIx = maxIx; + appendToNext = true; + } + } + + internal string GetString(int start, int limit) + { + if (limit <= start) + return ""; + if (start >= minIx && limit <= maxIx) + if (limit < brkIx) // String entirely in bldr builder + return bldr.ToString(start - minIx, limit - start); + else if (start >= brkIx) // String entirely in next builder + return next.ToString(start - brkIx, limit - start); + else // Must do a string-concatenation + return + bldr.ToString(start - minIx, brkIx - start) + + next.ToString(0, limit - brkIx); + else + throw new BufferException("String was outside data buffer"); + } + + internal void Mark(int limit) + { + if (limit > brkIx + 16) // Rotate blocks + { + StringBuilder temp = bldr; + bldr = next; + next = temp; + next.Length = 0; + minIx = brkIx; + brkIx = maxIx; + } + } + } + + BufferElement data = new BufferElement(); + + int bPos; // Postion index in the StringBuilder + BlockReader NextBlk; // Delegate that serves char-arrays; + + private string EncodingName + { + get + { + StreamReader rdr = NextBlk.Target as StreamReader; + return (rdr == null ? "raw-bytes" : rdr.CurrentEncoding.BodyName); + } + } + + public BuildBuffer(Stream stream) + { + FileStream fStrm = (stream as FileStream); + if (fStrm != null) FileName = fStrm.Name; + NextBlk = BlockReaderFactory.Raw(stream); + } + +#if (!BYTEMODE) + public BuildBuffer(Stream stream, int fallbackCodePage) + { + FileStream fStrm = (stream as FileStream); + if (fStrm != null) FileName = fStrm.Name; + NextBlk = BlockReaderFactory.Get(stream, fallbackCodePage); + } +#endif + + /// + /// Marks a conservative lower bound for the buffer, + /// allowing space to be reclaimed. If an application + /// needs to call GetString at arbitrary past locations + /// in the input stream, Mark() is not called. + /// + public override void Mark() { data.Mark(bPos - 2); } + + public override int Pos + { + get { return bPos; } + set { bPos = value; } + } + + + /// + /// Read returns the ordinal number of the next char, or + /// EOF (-1) for an end of stream. Note that the next + /// code point may require *two* calls of Read(). + /// + /// + public override int Read() + { + // + // Characters at positions + // [data.offset, data.offset + data.bldr.Length) + // are available in data.bldr. + // + if (bPos < data.MaxIndex) + { + // ch0 cannot be EOF + return (int)data[bPos++]; + } + else // Read from underlying stream + { + // Experimental code, blocks of page size + char[] chrs = new char[4096]; + int count = NextBlk(chrs, 0, 4096); + if (count == 0) + return EndOfFile; + else + { + data.Append(chrs, count); + return (int)data[bPos++]; + } + } + } + + public override string GetString(int begin, int limit) + { + return data.GetString(begin, limit); + } + + public override string ToString() + { + return "StringBuilder buffer, encoding: " + this.EncodingName; + } + } + + // =============== End ScanBuff-derived classes ================== + + internal delegate int BlockReader(char[] block, int index, int number); + + // A delegate factory, serving up a delegate that + // reads a block of characters from the underlying + // encoded stream, via a StreamReader object. + // + internal static class BlockReaderFactory + { + public static BlockReader Raw(Stream stream) + { + return delegate(char[] block, int index, int number) + { + byte[] b = new byte[number]; + int count = stream.Read(b, 0, number); + int i = 0; + int j = index; + for (; i < count; i++, j++) + block[j] = (char)b[i]; + return count; + }; + } + +#if (!BYTEMODE) + public static BlockReader Get(Stream stream, int fallbackCodePage) + { + Encoding encoding; + int preamble = Preamble(stream); + + if (preamble != 0) // There is a valid BOM here! + encoding = Encoding.GetEncoding(preamble); + else if (fallbackCodePage == -1) // Fallback is "raw" bytes + return Raw(stream); + else if (fallbackCodePage != -2) // Anything but "guess" + encoding = Encoding.GetEncoding(fallbackCodePage); + else // This is the "guess" option + { + int guess = new Guesser(stream).GuessCodePage(); + stream.Seek(0, SeekOrigin.Begin); + if (guess == -1) // ==> this is a 7-bit file + encoding = Encoding.ASCII; + else if (guess == 65001) + encoding = Encoding.UTF8; + else // ==> use the machine default + encoding = Encoding.Default; + } + StreamReader reader = new StreamReader(stream, encoding); + return reader.Read; + } + + static int Preamble(Stream stream) + { + int b0 = stream.ReadByte(); + int b1 = stream.ReadByte(); + + if (b0 == 0xfe && b1 == 0xff) + return 1201; // UTF16BE + if (b0 == 0xff && b1 == 0xfe) + return 1200; // UTF16LE + + int b2 = stream.ReadByte(); + if (b0 == 0xef && b1 == 0xbb && b2 == 0xbf) + return 65001; // UTF8 + // + // There is no unicode preamble, so we + // return denoter for the machine default. + // + stream.Seek(0, SeekOrigin.Begin); + return 0; + } +#endif // !BYTEMODE + } +#endif // !NOFILES + #endregion Buffer classes + + // ============================================================== + // ============ class CodePageHandling ============= + // ============================================================== +#if (!NOFILES) + public static class CodePageHandling + { + public static int GetCodePage(string option) + { + string command = option.ToUpperInvariant(); + if (command.StartsWith("CodePage:", StringComparison.OrdinalIgnoreCase)) + command = command.Substring(9); + try + { + if (command.Equals("RAW")) + return -1; + else if (command.Equals("GUESS")) + return -2; + else if (command.Equals("DEFAULT")) + return 0; + else if (char.IsDigit(command[0])) + return int.Parse(command, CultureInfo.InvariantCulture); + else + { + Encoding enc = Encoding.GetEncoding(command); + return enc.CodePage; + } + } + catch (FormatException) + { + Console.Error.WriteLine( + "Invalid format \"{0}\", using machine default", option); + } + catch (ArgumentException) + { + Console.Error.WriteLine( + "Unknown code page \"{0}\", using machine default", option); + } + return 0; + } + } +#region guesser +#if (!BYTEMODE) + // ============================================================== + // ============ Encoding Guesser ============= + // ============================================================== + + /// + /// This class provides a simple finite state automaton that + /// scans the file looking for (1) valid UTF-8 byte patterns, + /// (2) bytes >= 0x80 which are not part of a UTF-8 sequence. + /// The method then guesses whether it is UTF-8 or maybe some + /// local machine default encoding. This works well for the + /// various Latin encodings. + /// + internal class Guesser + { + ScanBuff buffer; + + public int GuessCodePage() { return Scan(); } + + const int maxAccept = 10; + const int initial = 0; + const int eofNum = 0; + const int goStart = -1; + const int INITIAL = 0; + const int EndToken = 0; + + #region user code + /* + * Reads the bytes of a file to determine if it is + * UTF-8 or a single-byte code page file. + */ + public long utfX; + public long uppr; + #endregion user code + + int state; + int currentStart = startState[0]; + int code; + + #region ScannerTables + static int[] startState = new int[] { 11, 0 }; + + #region CharacterMap + static sbyte[] map = new sbyte[256] { +/* '\0' */ 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, +/* '\x10' */ 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, +/* '\x20' */ 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, +/* '0' */ 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, +/* '@' */ 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, +/* 'P' */ 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, +/* '`' */ 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, +/* 'p' */ 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, +/* '\x80' */ 2, 2, 2, 2, 2, 2, 2, 2, 2, 2, 2, 2, 2, 2, 2, 2, +/* '\x90' */ 2, 2, 2, 2, 2, 2, 2, 2, 2, 2, 2, 2, 2, 2, 2, 2, +/* '\xA0' */ 2, 2, 2, 2, 2, 2, 2, 2, 2, 2, 2, 2, 2, 2, 2, 2, +/* '\xB0' */ 2, 2, 2, 2, 2, 2, 2, 2, 2, 2, 2, 2, 2, 2, 2, 2, +/* '\xC0' */ 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, +/* '\xD0' */ 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, +/* '\xE0' */ 3, 3, 3, 3, 3, 3, 3, 3, 3, 3, 3, 3, 3, 3, 3, 3, +/* '\xF0' */ 4, 4, 4, 4, 4, 4, 4, 4, 5, 5, 5, 5, 5, 5, 5, 5 }; + #endregion + + static sbyte[][] nextState = new sbyte[][] { + new sbyte[] {0, 0, 0, 0, 0, 0}, + new sbyte[] {-1, -1, 10, -1, -1, -1}, + new sbyte[] {-1, -1, -1, -1, -1, -1}, + new sbyte[] {-1, -1, 8, -1, -1, -1}, + new sbyte[] {-1, -1, 5, -1, -1, -1}, + new sbyte[] {-1, -1, 6, -1, -1, -1}, + new sbyte[] {-1, -1, 7, -1, -1, -1}, + null, + new sbyte[] {-1, -1, 9, -1, -1, -1}, + null, + null, + new sbyte[] {-1, 1, 2, 3, 4, 2} + }; + + + [SuppressMessage("Microsoft.Performance", "CA1810:InitializeReferenceTypeStaticFieldsInline")] + // Reason for suppression: cannot have self-reference in array initializer. + static Guesser() + { + nextState[7] = nextState[2]; + nextState[9] = nextState[2]; + nextState[10] = nextState[2]; + } + + int NextState() + { + if (code == ScanBuff.EndOfFile) + return eofNum; + else + return nextState[state][map[code]]; + } + #endregion + + public Guesser(System.IO.Stream file) { SetSource(file); } + + public void SetSource(System.IO.Stream source) + { + this.buffer = new BuildBuffer(source); + code = buffer.Read(); + } + + int Scan() + { + for (; ; ) + { + int next; + state = currentStart; + while ((next = NextState()) == goStart) + code = buffer.Read(); + + state = next; + code = buffer.Read(); + + while ((next = NextState()) > eofNum) + { + state = next; + code = buffer.Read(); + } + if (state <= maxAccept) + { + #region ActionSwitch +#pragma warning disable 162 + switch (state) + { + case eofNum: + switch (currentStart) + { + case 11: + if (utfX == 0 && uppr == 0) return -1; /* raw ascii */ + else if (uppr * 10 > utfX) return 0; /* default code page */ + else return 65001; /* UTF-8 encoding */ + break; + } + return EndToken; + case 1: // Recognized '{Upper128}', Shortest string "\xC0" + case 2: // Recognized '{Upper128}', Shortest string "\x80" + case 3: // Recognized '{Upper128}', Shortest string "\xE0" + case 4: // Recognized '{Upper128}', Shortest string "\xF0" + uppr++; + break; + case 5: // Recognized '{Utf8pfx4}{Utf8cont}', Shortest string "\xF0\x80" + uppr += 2; + break; + case 6: // Recognized '{Utf8pfx4}{Utf8cont}{2}', Shortest string "\xF0\x80\x80" + uppr += 3; + break; + case 7: // Recognized '{Utf8pfx4}{Utf8cont}{3}', Shortest string "\xF0\x80\x80\x80" + utfX += 3; + break; + case 8: // Recognized '{Utf8pfx3}{Utf8cont}', Shortest string "\xE0\x80" + uppr += 2; + break; + case 9: // Recognized '{Utf8pfx3}{Utf8cont}{2}', Shortest string "\xE0\x80\x80" + utfX += 2; + break; + case 10: // Recognized '{Utf8pfx2}{Utf8cont}', Shortest string "\xC0\x80" + utfX++; + break; + default: + break; + } +#pragma warning restore 162 + #endregion + } + } + } + } // end class Guesser + +#endif // !BYTEMODE +#endregion +#endif // !NOFILES + +// End of code copied from embedded resource +} diff --git a/src/DataStreamSuite/BooleanExpressions/GplexBuffers.cs b/src/DataStreamSuite/BooleanExpressions/GplexBuffers.cs index 76dd35443..1bcc08e09 100644 --- a/src/DataStreamSuite/BooleanExpressions/GplexBuffers.cs +++ b/src/DataStreamSuite/BooleanExpressions/GplexBuffers.cs @@ -1,720 +1,720 @@ -// ============================================================== -// -// This code automatically produced from an embedded resource. -// Do not edit this file, or it will become incompatible with -// the specification from which it was generated. -// -// ============================================================== - -using System; -using System.IO; -using System.Text; -using System.Collections.Generic; -using System.Diagnostics.CodeAnalysis; -using System.Runtime.Serialization; -using System.Globalization; - -namespace QUT.GplexBuffers -{ -// Code copied from GPLEX embedded resource - [Serializable] - internal class BufferException : Exception - { - public BufferException() { } - public BufferException(string message) : base(message) { } - public BufferException(string message, Exception innerException) - : base(message, innerException) { } - protected BufferException(SerializationInfo info, StreamingContext context) - : base(info, context) { } - } - - internal abstract class ScanBuff - { - private string fileNm; - - public const int EndOfFile = -1; - public const int UnicodeReplacementChar = 0xFFFD; - - public bool IsFile { get { return (fileNm != null); } } - public string FileName { get { return fileNm; } set { fileNm = value; } } - - public abstract int Pos { get; set; } - public abstract int Read(); - public virtual void Mark() { } - - public abstract string GetString(int begin, int limit); - - public static ScanBuff GetBuffer(string source) - { - return new StringBuffer(source); - } - - public static ScanBuff GetBuffer(IList source) - { - return new LineBuffer(source); - } - -#if (!NOFILES) - public static ScanBuff GetBuffer(Stream source) - { - return new BuildBuffer(source); - } - -#if (!BYTEMODE) - public static ScanBuff GetBuffer(Stream source, int fallbackCodePage) - { - return new BuildBuffer(source, fallbackCodePage); - } -#endif // !BYTEMODE -#endif // !NOFILES - } - - #region Buffer classes - - // ============================================================== - // ===== Definitions for various ScanBuff derived classes ==== - // ============================================================== - // =============== String input ================ - // ============================================================== - - /// - /// This class reads characters from a single string as - /// required, for example, by Visual Studio language services - /// - sealed class StringBuffer : ScanBuff - { - string str; // input buffer - int bPos; // current position in buffer - int sLen; - - public StringBuffer(string source) - { - this.str = source; - this.sLen = source.Length; - this.FileName = null; - } - - public override int Read() - { - if (bPos < sLen) return str[bPos++]; - else if (bPos == sLen) { bPos++; return '\n'; } // one strike, see new line - else { bPos++; return EndOfFile; } // two strikes and you're out! - } - - public override string GetString(int begin, int limit) - { - // "limit" can be greater than sLen with the BABEL - // option set. Read returns a "virtual" EOL if - // an attempt is made to read past the end of the - // string buffer. Without the guard any attempt - // to fetch yytext for a token that includes the - // EOL will throw an index exception. - if (limit > sLen) limit = sLen; - if (limit <= begin) return ""; - else return str.Substring(begin, limit - begin); - } - - public override int Pos - { - get { return bPos; } - set { bPos = value; } - } - - public override string ToString() { return "StringBuffer"; } - } - - // ============================================================== - // The LineBuff class contributed by Nigel Horspool, - // nigelh@cs.uvic.cs - // ============================================================== - - sealed class LineBuffer : ScanBuff - { - IList line; // list of source lines from a file - int numLines; // number of strings in line list - string curLine; // current line in that list - int cLine; // index of current line in the list - int curLen; // length of current line - int curLineStart; // position of line start in whole file - int curLineEnd; // position of line end in whole file - int maxPos; // max position ever visited in whole file - int cPos; // ordinal number of code in source - - // Constructed from a list of strings, one per source line. - // The lines have had trailing '\n' characters removed. - public LineBuffer(IList lineList) - { - line = lineList; - numLines = line.Count; - cPos = curLineStart = 0; - curLine = (numLines > 0 ? line[0] : ""); - maxPos = curLineEnd = curLen = curLine.Length; - cLine = 1; - FileName = null; - } - - public override int Read() - { - if (cPos < curLineEnd) - return curLine[cPos++ - curLineStart]; - if (cPos++ == curLineEnd) - return '\n'; - if (cLine >= numLines) - return EndOfFile; - curLine = line[cLine]; - curLen = curLine.Length; - curLineStart = curLineEnd + 1; - curLineEnd = curLineStart + curLen; - if (curLineEnd > maxPos) - maxPos = curLineEnd; - cLine++; - return curLen > 0 ? curLine[0] : '\n'; - } - - // To speed up searches for the line containing a position - private int cachedPosition; - private int cachedIxdex; - private int cachedLineStart; - - // Given a position pos within the entire source, the results are - // ix -- the index of the containing line - // lstart -- the position of the first character on that line - private void findIndex(int pos, out int ix, out int lstart) - { - if (pos >= cachedPosition) - { - ix = cachedIxdex; lstart = cachedLineStart; - } - else - { - ix = lstart = 0; - } - while (ix < numLines) - { - int len = line[ix].Length + 1; - if (pos < lstart + len) break; - lstart += len; - ix++; - } - cachedPosition = pos; - cachedIxdex = ix; - cachedLineStart = lstart; - } - - public override string GetString(int begin, int limit) - { - if (begin >= maxPos || limit <= begin) return ""; - int endIx, begIx, endLineStart, begLineStart; - findIndex(begin, out begIx, out begLineStart); - int begCol = begin - begLineStart; - findIndex(limit, out endIx, out endLineStart); - int endCol = limit - endLineStart; - string s = line[begIx]; - if (begIx == endIx) - { - // the usual case, substring all on one line - return (endCol <= s.Length) ? - s.Substring(begCol, endCol - begCol) - : s.Substring(begCol) + "\n"; - } - // the string spans multiple lines, yuk! - StringBuilder sb = new StringBuilder(); - if (begCol < s.Length) - sb.Append(s.Substring(begCol)); - for (; ; ) - { - sb.Append("\n"); - s = line[++begIx]; - if (begIx >= endIx) break; - sb.Append(s); - } - if (endCol <= s.Length) - { - sb.Append(s.Substring(0, endCol)); - } - else - { - sb.Append(s); - sb.Append("\n"); - } - return sb.ToString(); - } - - public override int Pos - { - get { return cPos; } - set - { - cPos = value; - findIndex(cPos, out cLine, out curLineStart); - // cLine should be the *next* line after curLine. - curLine = (cLine < numLines ? line[cLine++] : ""); - curLineEnd = curLineStart + curLine.Length; - } - } - - public override string ToString() { return "LineBuffer"; } - } - -#if (!NOFILES) - // ============================================================== - // ===== class BuildBuff : for unicode text files ======== - // ============================================================== - - class BuildBuffer : ScanBuff - { - // Double buffer for char stream. - class BufferElement - { - StringBuilder bldr = new StringBuilder(); - StringBuilder next = new StringBuilder(); - int minIx; - int maxIx; - int brkIx; - bool appendToNext; - - internal BufferElement() { } - - internal int MaxIndex { get { return maxIx; } } - // internal int MinIndex { get { return minIx; } } - - internal char this[int index] - { - get - { - if (index < minIx || index >= maxIx) - throw new BufferException("Index was outside data buffer"); - else if (index < brkIx) - return bldr[index - minIx]; - else - return next[index - brkIx]; - } - } - - internal void Append(char[] block, int count) - { - maxIx += count; - if (appendToNext) - this.next.Append(block, 0, count); - else - { - this.bldr.Append(block, 0, count); - brkIx = maxIx; - appendToNext = true; - } - } - - internal string GetString(int start, int limit) - { - if (limit <= start) - return ""; - if (start >= minIx && limit <= maxIx) - if (limit < brkIx) // String entirely in bldr builder - return bldr.ToString(start - minIx, limit - start); - else if (start >= brkIx) // String entirely in next builder - return next.ToString(start - brkIx, limit - start); - else // Must do a string-concatenation - return - bldr.ToString(start - minIx, brkIx - start) + - next.ToString(0, limit - brkIx); - else - throw new BufferException("String was outside data buffer"); - } - - internal void Mark(int limit) - { - if (limit > brkIx + 16) // Rotate blocks - { - StringBuilder temp = bldr; - bldr = next; - next = temp; - next.Length = 0; - minIx = brkIx; - brkIx = maxIx; - } - } - } - - BufferElement data = new BufferElement(); - - int bPos; // Postion index in the StringBuilder - BlockReader NextBlk; // Delegate that serves char-arrays; - - private string EncodingName - { - get - { - StreamReader rdr = NextBlk.Target as StreamReader; - return (rdr == null ? "raw-bytes" : rdr.CurrentEncoding.BodyName); - } - } - - public BuildBuffer(Stream stream) - { - FileStream fStrm = (stream as FileStream); - if (fStrm != null) FileName = fStrm.Name; - NextBlk = BlockReaderFactory.Raw(stream); - } - -#if (!BYTEMODE) - public BuildBuffer(Stream stream, int fallbackCodePage) - { - FileStream fStrm = (stream as FileStream); - if (fStrm != null) FileName = fStrm.Name; - NextBlk = BlockReaderFactory.Get(stream, fallbackCodePage); - } -#endif - - /// - /// Marks a conservative lower bound for the buffer, - /// allowing space to be reclaimed. If an application - /// needs to call GetString at arbitrary past locations - /// in the input stream, Mark() is not called. - /// - public override void Mark() { data.Mark(bPos - 2); } - - public override int Pos - { - get { return bPos; } - set { bPos = value; } - } - - - /// - /// Read returns the ordinal number of the next char, or - /// EOF (-1) for an end of stream. Note that the next - /// code point may require *two* calls of Read(). - /// - /// - public override int Read() - { - // - // Characters at positions - // [data.offset, data.offset + data.bldr.Length) - // are available in data.bldr. - // - if (bPos < data.MaxIndex) - { - // ch0 cannot be EOF - return (int)data[bPos++]; - } - else // Read from underlying stream - { - // Experimental code, blocks of page size - char[] chrs = new char[4096]; - int count = NextBlk(chrs, 0, 4096); - if (count == 0) - return EndOfFile; - else - { - data.Append(chrs, count); - return (int)data[bPos++]; - } - } - } - - public override string GetString(int begin, int limit) - { - return data.GetString(begin, limit); - } - - public override string ToString() - { - return "StringBuilder buffer, encoding: " + this.EncodingName; - } - } - - // =============== End ScanBuff-derived classes ================== - - internal delegate int BlockReader(char[] block, int index, int number); - - // A delegate factory, serving up a delegate that - // reads a block of characters from the underlying - // encoded stream, via a StreamReader object. - // - internal static class BlockReaderFactory - { - public static BlockReader Raw(Stream stream) - { - return delegate(char[] block, int index, int number) - { - byte[] b = new byte[number]; - int count = stream.Read(b, 0, number); - int i = 0; - int j = index; - for (; i < count; i++, j++) - block[j] = (char)b[i]; - return count; - }; - } - -#if (!BYTEMODE) - public static BlockReader Get(Stream stream, int fallbackCodePage) - { - Encoding encoding; - int preamble = Preamble(stream); - - if (preamble != 0) // There is a valid BOM here! - encoding = Encoding.GetEncoding(preamble); - else if (fallbackCodePage == -1) // Fallback is "raw" bytes - return Raw(stream); - else if (fallbackCodePage != -2) // Anything but "guess" - encoding = Encoding.GetEncoding(fallbackCodePage); - else // This is the "guess" option - { - int guess = new Guesser(stream).GuessCodePage(); - stream.Seek(0, SeekOrigin.Begin); - if (guess == -1) // ==> this is a 7-bit file - encoding = Encoding.ASCII; - else if (guess == 65001) - encoding = Encoding.UTF8; - else // ==> use the machine default - encoding = Encoding.Default; - } - StreamReader reader = new StreamReader(stream, encoding); - return reader.Read; - } - - static int Preamble(Stream stream) - { - int b0 = stream.ReadByte(); - int b1 = stream.ReadByte(); - - if (b0 == 0xfe && b1 == 0xff) - return 1201; // UTF16BE - if (b0 == 0xff && b1 == 0xfe) - return 1200; // UTF16LE - - int b2 = stream.ReadByte(); - if (b0 == 0xef && b1 == 0xbb && b2 == 0xbf) - return 65001; // UTF8 - // - // There is no unicode preamble, so we - // return denoter for the machine default. - // - stream.Seek(0, SeekOrigin.Begin); - return 0; - } -#endif // !BYTEMODE - } -#endif // !NOFILES - #endregion Buffer classes - - // ============================================================== - // ============ class CodePageHandling ============= - // ============================================================== -#if (!NOFILES) - public static class CodePageHandling - { - public static int GetCodePage(string option) - { - string command = option.ToUpperInvariant(); - if (command.StartsWith("CodePage:", StringComparison.OrdinalIgnoreCase)) - command = command.Substring(9); - try - { - if (command.Equals("RAW")) - return -1; - else if (command.Equals("GUESS")) - return -2; - else if (command.Equals("DEFAULT")) - return 0; - else if (char.IsDigit(command[0])) - return int.Parse(command, CultureInfo.InvariantCulture); - else - { - Encoding enc = Encoding.GetEncoding(command); - return enc.CodePage; - } - } - catch (FormatException) - { - Console.Error.WriteLine( - "Invalid format \"{0}\", using machine default", option); - } - catch (ArgumentException) - { - Console.Error.WriteLine( - "Unknown code page \"{0}\", using machine default", option); - } - return 0; - } - } -#region guesser -#if (!BYTEMODE) - // ============================================================== - // ============ Encoding Guesser ============= - // ============================================================== - - /// - /// This class provides a simple finite state automaton that - /// scans the file looking for (1) valid UTF-8 byte patterns, - /// (2) bytes >= 0x80 which are not part of a UTF-8 sequence. - /// The method then guesses whether it is UTF-8 or maybe some - /// local machine default encoding. This works well for the - /// various Latin encodings. - /// - internal class Guesser - { - ScanBuff buffer; - - public int GuessCodePage() { return Scan(); } - - const int maxAccept = 10; - const int initial = 0; - const int eofNum = 0; - const int goStart = -1; - const int INITIAL = 0; - const int EndToken = 0; - - #region user code - /* - * Reads the bytes of a file to determine if it is - * UTF-8 or a single-byte code page file. - */ - public long utfX; - public long uppr; - #endregion user code - - int state; - int currentStart = startState[0]; - int code; - - #region ScannerTables - static int[] startState = new int[] { 11, 0 }; - - #region CharacterMap - static sbyte[] map = new sbyte[256] { -/* '\0' */ 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, -/* '\x10' */ 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, -/* '\x20' */ 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, -/* '0' */ 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, -/* '@' */ 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, -/* 'P' */ 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, -/* '`' */ 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, -/* 'p' */ 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, -/* '\x80' */ 2, 2, 2, 2, 2, 2, 2, 2, 2, 2, 2, 2, 2, 2, 2, 2, -/* '\x90' */ 2, 2, 2, 2, 2, 2, 2, 2, 2, 2, 2, 2, 2, 2, 2, 2, -/* '\xA0' */ 2, 2, 2, 2, 2, 2, 2, 2, 2, 2, 2, 2, 2, 2, 2, 2, -/* '\xB0' */ 2, 2, 2, 2, 2, 2, 2, 2, 2, 2, 2, 2, 2, 2, 2, 2, -/* '\xC0' */ 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, -/* '\xD0' */ 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, -/* '\xE0' */ 3, 3, 3, 3, 3, 3, 3, 3, 3, 3, 3, 3, 3, 3, 3, 3, -/* '\xF0' */ 4, 4, 4, 4, 4, 4, 4, 4, 5, 5, 5, 5, 5, 5, 5, 5 }; - #endregion - - static sbyte[][] nextState = new sbyte[][] { - new sbyte[] {0, 0, 0, 0, 0, 0}, - new sbyte[] {-1, -1, 10, -1, -1, -1}, - new sbyte[] {-1, -1, -1, -1, -1, -1}, - new sbyte[] {-1, -1, 8, -1, -1, -1}, - new sbyte[] {-1, -1, 5, -1, -1, -1}, - new sbyte[] {-1, -1, 6, -1, -1, -1}, - new sbyte[] {-1, -1, 7, -1, -1, -1}, - null, - new sbyte[] {-1, -1, 9, -1, -1, -1}, - null, - null, - new sbyte[] {-1, 1, 2, 3, 4, 2} - }; - - - [SuppressMessage("Microsoft.Performance", "CA1810:InitializeReferenceTypeStaticFieldsInline")] - // Reason for suppression: cannot have self-reference in array initializer. - static Guesser() - { - nextState[7] = nextState[2]; - nextState[9] = nextState[2]; - nextState[10] = nextState[2]; - } - - int NextState() - { - if (code == ScanBuff.EndOfFile) - return eofNum; - else - return nextState[state][map[code]]; - } - #endregion - - public Guesser(System.IO.Stream file) { SetSource(file); } - - public void SetSource(System.IO.Stream source) - { - this.buffer = new BuildBuffer(source); - code = buffer.Read(); - } - - int Scan() - { - for (; ; ) - { - int next; - state = currentStart; - while ((next = NextState()) == goStart) - code = buffer.Read(); - - state = next; - code = buffer.Read(); - - while ((next = NextState()) > eofNum) - { - state = next; - code = buffer.Read(); - } - if (state <= maxAccept) - { - #region ActionSwitch -#pragma warning disable 162 - switch (state) - { - case eofNum: - switch (currentStart) - { - case 11: - if (utfX == 0 && uppr == 0) return -1; /* raw ascii */ - else if (uppr * 10 > utfX) return 0; /* default code page */ - else return 65001; /* UTF-8 encoding */ - break; - } - return EndToken; - case 1: // Recognized '{Upper128}', Shortest string "\xC0" - case 2: // Recognized '{Upper128}', Shortest string "\x80" - case 3: // Recognized '{Upper128}', Shortest string "\xE0" - case 4: // Recognized '{Upper128}', Shortest string "\xF0" - uppr++; - break; - case 5: // Recognized '{Utf8pfx4}{Utf8cont}', Shortest string "\xF0\x80" - uppr += 2; - break; - case 6: // Recognized '{Utf8pfx4}{Utf8cont}{2}', Shortest string "\xF0\x80\x80" - uppr += 3; - break; - case 7: // Recognized '{Utf8pfx4}{Utf8cont}{3}', Shortest string "\xF0\x80\x80\x80" - utfX += 3; - break; - case 8: // Recognized '{Utf8pfx3}{Utf8cont}', Shortest string "\xE0\x80" - uppr += 2; - break; - case 9: // Recognized '{Utf8pfx3}{Utf8cont}{2}', Shortest string "\xE0\x80\x80" - utfX += 2; - break; - case 10: // Recognized '{Utf8pfx2}{Utf8cont}', Shortest string "\xC0\x80" - utfX++; - break; - default: - break; - } -#pragma warning restore 162 - #endregion - } - } - } - } // end class Guesser - -#endif // !BYTEMODE -#endregion -#endif // !NOFILES - -// End of code copied from embedded resource -} +// ============================================================== +// +// This code automatically produced from an embedded resource. +// Do not edit this file, or it will become incompatible with +// the specification from which it was generated. +// +// ============================================================== + +using System; +using System.IO; +using System.Text; +using System.Collections.Generic; +using System.Diagnostics.CodeAnalysis; +using System.Runtime.Serialization; +using System.Globalization; + +namespace QUT.GplexBuffers +{ +// Code copied from GPLEX embedded resource + [Serializable] + internal class BufferException : Exception + { + public BufferException() { } + public BufferException(string message) : base(message) { } + public BufferException(string message, Exception innerException) + : base(message, innerException) { } + protected BufferException(SerializationInfo info, StreamingContext context) + : base(info, context) { } + } + + internal abstract class ScanBuff + { + private string fileNm; + + public const int EndOfFile = -1; + public const int UnicodeReplacementChar = 0xFFFD; + + public bool IsFile { get { return (fileNm != null); } } + public string FileName { get { return fileNm; } set { fileNm = value; } } + + public abstract int Pos { get; set; } + public abstract int Read(); + public virtual void Mark() { } + + public abstract string GetString(int begin, int limit); + + public static ScanBuff GetBuffer(string source) + { + return new StringBuffer(source); + } + + public static ScanBuff GetBuffer(IList source) + { + return new LineBuffer(source); + } + +#if (!NOFILES) + public static ScanBuff GetBuffer(Stream source) + { + return new BuildBuffer(source); + } + +#if (!BYTEMODE) + public static ScanBuff GetBuffer(Stream source, int fallbackCodePage) + { + return new BuildBuffer(source, fallbackCodePage); + } +#endif // !BYTEMODE +#endif // !NOFILES + } + + #region Buffer classes + + // ============================================================== + // ===== Definitions for various ScanBuff derived classes ==== + // ============================================================== + // =============== String input ================ + // ============================================================== + + /// + /// This class reads characters from a single string as + /// required, for example, by Visual Studio language services + /// + sealed class StringBuffer : ScanBuff + { + string str; // input buffer + int bPos; // current position in buffer + int sLen; + + public StringBuffer(string source) + { + this.str = source; + this.sLen = source.Length; + this.FileName = null; + } + + public override int Read() + { + if (bPos < sLen) return str[bPos++]; + else if (bPos == sLen) { bPos++; return '\n'; } // one strike, see new line + else { bPos++; return EndOfFile; } // two strikes and you're out! + } + + public override string GetString(int begin, int limit) + { + // "limit" can be greater than sLen with the BABEL + // option set. Read returns a "virtual" EOL if + // an attempt is made to read past the end of the + // string buffer. Without the guard any attempt + // to fetch yytext for a token that includes the + // EOL will throw an index exception. + if (limit > sLen) limit = sLen; + if (limit <= begin) return ""; + else return str.Substring(begin, limit - begin); + } + + public override int Pos + { + get { return bPos; } + set { bPos = value; } + } + + public override string ToString() { return "StringBuffer"; } + } + + // ============================================================== + // The LineBuff class contributed by Nigel Horspool, + // nigelh@cs.uvic.cs + // ============================================================== + + sealed class LineBuffer : ScanBuff + { + IList line; // list of source lines from a file + int numLines; // number of strings in line list + string curLine; // current line in that list + int cLine; // index of current line in the list + int curLen; // length of current line + int curLineStart; // position of line start in whole file + int curLineEnd; // position of line end in whole file + int maxPos; // max position ever visited in whole file + int cPos; // ordinal number of code in source + + // Constructed from a list of strings, one per source line. + // The lines have had trailing '\n' characters removed. + public LineBuffer(IList lineList) + { + line = lineList; + numLines = line.Count; + cPos = curLineStart = 0; + curLine = (numLines > 0 ? line[0] : ""); + maxPos = curLineEnd = curLen = curLine.Length; + cLine = 1; + FileName = null; + } + + public override int Read() + { + if (cPos < curLineEnd) + return curLine[cPos++ - curLineStart]; + if (cPos++ == curLineEnd) + return '\n'; + if (cLine >= numLines) + return EndOfFile; + curLine = line[cLine]; + curLen = curLine.Length; + curLineStart = curLineEnd + 1; + curLineEnd = curLineStart + curLen; + if (curLineEnd > maxPos) + maxPos = curLineEnd; + cLine++; + return curLen > 0 ? curLine[0] : '\n'; + } + + // To speed up searches for the line containing a position + private int cachedPosition; + private int cachedIxdex; + private int cachedLineStart; + + // Given a position pos within the entire source, the results are + // ix -- the index of the containing line + // lstart -- the position of the first character on that line + private void findIndex(int pos, out int ix, out int lstart) + { + if (pos >= cachedPosition) + { + ix = cachedIxdex; lstart = cachedLineStart; + } + else + { + ix = lstart = 0; + } + while (ix < numLines) + { + int len = line[ix].Length + 1; + if (pos < lstart + len) break; + lstart += len; + ix++; + } + cachedPosition = pos; + cachedIxdex = ix; + cachedLineStart = lstart; + } + + public override string GetString(int begin, int limit) + { + if (begin >= maxPos || limit <= begin) return ""; + int endIx, begIx, endLineStart, begLineStart; + findIndex(begin, out begIx, out begLineStart); + int begCol = begin - begLineStart; + findIndex(limit, out endIx, out endLineStart); + int endCol = limit - endLineStart; + string s = line[begIx]; + if (begIx == endIx) + { + // the usual case, substring all on one line + return (endCol <= s.Length) ? + s.Substring(begCol, endCol - begCol) + : s.Substring(begCol) + "\n"; + } + // the string spans multiple lines, yuk! + StringBuilder sb = new StringBuilder(); + if (begCol < s.Length) + sb.Append(s.Substring(begCol)); + for (; ; ) + { + sb.Append("\n"); + s = line[++begIx]; + if (begIx >= endIx) break; + sb.Append(s); + } + if (endCol <= s.Length) + { + sb.Append(s.Substring(0, endCol)); + } + else + { + sb.Append(s); + sb.Append("\n"); + } + return sb.ToString(); + } + + public override int Pos + { + get { return cPos; } + set + { + cPos = value; + findIndex(cPos, out cLine, out curLineStart); + // cLine should be the *next* line after curLine. + curLine = (cLine < numLines ? line[cLine++] : ""); + curLineEnd = curLineStart + curLine.Length; + } + } + + public override string ToString() { return "LineBuffer"; } + } + +#if (!NOFILES) + // ============================================================== + // ===== class BuildBuff : for unicode text files ======== + // ============================================================== + + class BuildBuffer : ScanBuff + { + // Double buffer for char stream. + class BufferElement + { + StringBuilder bldr = new StringBuilder(); + StringBuilder next = new StringBuilder(); + int minIx; + int maxIx; + int brkIx; + bool appendToNext; + + internal BufferElement() { } + + internal int MaxIndex { get { return maxIx; } } + // internal int MinIndex { get { return minIx; } } + + internal char this[int index] + { + get + { + if (index < minIx || index >= maxIx) + throw new BufferException("Index was outside data buffer"); + else if (index < brkIx) + return bldr[index - minIx]; + else + return next[index - brkIx]; + } + } + + internal void Append(char[] block, int count) + { + maxIx += count; + if (appendToNext) + this.next.Append(block, 0, count); + else + { + this.bldr.Append(block, 0, count); + brkIx = maxIx; + appendToNext = true; + } + } + + internal string GetString(int start, int limit) + { + if (limit <= start) + return ""; + if (start >= minIx && limit <= maxIx) + if (limit < brkIx) // String entirely in bldr builder + return bldr.ToString(start - minIx, limit - start); + else if (start >= brkIx) // String entirely in next builder + return next.ToString(start - brkIx, limit - start); + else // Must do a string-concatenation + return + bldr.ToString(start - minIx, brkIx - start) + + next.ToString(0, limit - brkIx); + else + throw new BufferException("String was outside data buffer"); + } + + internal void Mark(int limit) + { + if (limit > brkIx + 16) // Rotate blocks + { + StringBuilder temp = bldr; + bldr = next; + next = temp; + next.Length = 0; + minIx = brkIx; + brkIx = maxIx; + } + } + } + + BufferElement data = new BufferElement(); + + int bPos; // Postion index in the StringBuilder + BlockReader NextBlk; // Delegate that serves char-arrays; + + private string EncodingName + { + get + { + StreamReader rdr = NextBlk.Target as StreamReader; + return (rdr == null ? "raw-bytes" : rdr.CurrentEncoding.BodyName); + } + } + + public BuildBuffer(Stream stream) + { + FileStream fStrm = (stream as FileStream); + if (fStrm != null) FileName = fStrm.Name; + NextBlk = BlockReaderFactory.Raw(stream); + } + +#if (!BYTEMODE) + public BuildBuffer(Stream stream, int fallbackCodePage) + { + FileStream fStrm = (stream as FileStream); + if (fStrm != null) FileName = fStrm.Name; + NextBlk = BlockReaderFactory.Get(stream, fallbackCodePage); + } +#endif + + /// + /// Marks a conservative lower bound for the buffer, + /// allowing space to be reclaimed. If an application + /// needs to call GetString at arbitrary past locations + /// in the input stream, Mark() is not called. + /// + public override void Mark() { data.Mark(bPos - 2); } + + public override int Pos + { + get { return bPos; } + set { bPos = value; } + } + + + /// + /// Read returns the ordinal number of the next char, or + /// EOF (-1) for an end of stream. Note that the next + /// code point may require *two* calls of Read(). + /// + /// + public override int Read() + { + // + // Characters at positions + // [data.offset, data.offset + data.bldr.Length) + // are available in data.bldr. + // + if (bPos < data.MaxIndex) + { + // ch0 cannot be EOF + return (int)data[bPos++]; + } + else // Read from underlying stream + { + // Experimental code, blocks of page size + char[] chrs = new char[4096]; + int count = NextBlk(chrs, 0, 4096); + if (count == 0) + return EndOfFile; + else + { + data.Append(chrs, count); + return (int)data[bPos++]; + } + } + } + + public override string GetString(int begin, int limit) + { + return data.GetString(begin, limit); + } + + public override string ToString() + { + return "StringBuilder buffer, encoding: " + this.EncodingName; + } + } + + // =============== End ScanBuff-derived classes ================== + + internal delegate int BlockReader(char[] block, int index, int number); + + // A delegate factory, serving up a delegate that + // reads a block of characters from the underlying + // encoded stream, via a StreamReader object. + // + internal static class BlockReaderFactory + { + public static BlockReader Raw(Stream stream) + { + return delegate(char[] block, int index, int number) + { + byte[] b = new byte[number]; + int count = stream.Read(b, 0, number); + int i = 0; + int j = index; + for (; i < count; i++, j++) + block[j] = (char)b[i]; + return count; + }; + } + +#if (!BYTEMODE) + public static BlockReader Get(Stream stream, int fallbackCodePage) + { + Encoding encoding; + int preamble = Preamble(stream); + + if (preamble != 0) // There is a valid BOM here! + encoding = Encoding.GetEncoding(preamble); + else if (fallbackCodePage == -1) // Fallback is "raw" bytes + return Raw(stream); + else if (fallbackCodePage != -2) // Anything but "guess" + encoding = Encoding.GetEncoding(fallbackCodePage); + else // This is the "guess" option + { + int guess = new Guesser(stream).GuessCodePage(); + stream.Seek(0, SeekOrigin.Begin); + if (guess == -1) // ==> this is a 7-bit file + encoding = Encoding.ASCII; + else if (guess == 65001) + encoding = Encoding.UTF8; + else // ==> use the machine default + encoding = Encoding.Default; + } + StreamReader reader = new StreamReader(stream, encoding); + return reader.Read; + } + + static int Preamble(Stream stream) + { + int b0 = stream.ReadByte(); + int b1 = stream.ReadByte(); + + if (b0 == 0xfe && b1 == 0xff) + return 1201; // UTF16BE + if (b0 == 0xff && b1 == 0xfe) + return 1200; // UTF16LE + + int b2 = stream.ReadByte(); + if (b0 == 0xef && b1 == 0xbb && b2 == 0xbf) + return 65001; // UTF8 + // + // There is no unicode preamble, so we + // return denoter for the machine default. + // + stream.Seek(0, SeekOrigin.Begin); + return 0; + } +#endif // !BYTEMODE + } +#endif // !NOFILES + #endregion Buffer classes + + // ============================================================== + // ============ class CodePageHandling ============= + // ============================================================== +#if (!NOFILES) + public static class CodePageHandling + { + public static int GetCodePage(string option) + { + string command = option.ToUpperInvariant(); + if (command.StartsWith("CodePage:", StringComparison.OrdinalIgnoreCase)) + command = command.Substring(9); + try + { + if (command.Equals("RAW")) + return -1; + else if (command.Equals("GUESS")) + return -2; + else if (command.Equals("DEFAULT")) + return 0; + else if (char.IsDigit(command[0])) + return int.Parse(command, CultureInfo.InvariantCulture); + else + { + Encoding enc = Encoding.GetEncoding(command); + return enc.CodePage; + } + } + catch (FormatException) + { + Console.Error.WriteLine( + "Invalid format \"{0}\", using machine default", option); + } + catch (ArgumentException) + { + Console.Error.WriteLine( + "Unknown code page \"{0}\", using machine default", option); + } + return 0; + } + } +#region guesser +#if (!BYTEMODE) + // ============================================================== + // ============ Encoding Guesser ============= + // ============================================================== + + /// + /// This class provides a simple finite state automaton that + /// scans the file looking for (1) valid UTF-8 byte patterns, + /// (2) bytes >= 0x80 which are not part of a UTF-8 sequence. + /// The method then guesses whether it is UTF-8 or maybe some + /// local machine default encoding. This works well for the + /// various Latin encodings. + /// + internal class Guesser + { + ScanBuff buffer; + + public int GuessCodePage() { return Scan(); } + + const int maxAccept = 10; + const int initial = 0; + const int eofNum = 0; + const int goStart = -1; + const int INITIAL = 0; + const int EndToken = 0; + + #region user code + /* + * Reads the bytes of a file to determine if it is + * UTF-8 or a single-byte code page file. + */ + public long utfX; + public long uppr; + #endregion user code + + int state; + int currentStart = startState[0]; + int code; + + #region ScannerTables + static int[] startState = new int[] { 11, 0 }; + + #region CharacterMap + static sbyte[] map = new sbyte[256] { +/* '\0' */ 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, +/* '\x10' */ 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, +/* '\x20' */ 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, +/* '0' */ 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, +/* '@' */ 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, +/* 'P' */ 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, +/* '`' */ 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, +/* 'p' */ 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, +/* '\x80' */ 2, 2, 2, 2, 2, 2, 2, 2, 2, 2, 2, 2, 2, 2, 2, 2, +/* '\x90' */ 2, 2, 2, 2, 2, 2, 2, 2, 2, 2, 2, 2, 2, 2, 2, 2, +/* '\xA0' */ 2, 2, 2, 2, 2, 2, 2, 2, 2, 2, 2, 2, 2, 2, 2, 2, +/* '\xB0' */ 2, 2, 2, 2, 2, 2, 2, 2, 2, 2, 2, 2, 2, 2, 2, 2, +/* '\xC0' */ 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, +/* '\xD0' */ 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, +/* '\xE0' */ 3, 3, 3, 3, 3, 3, 3, 3, 3, 3, 3, 3, 3, 3, 3, 3, +/* '\xF0' */ 4, 4, 4, 4, 4, 4, 4, 4, 5, 5, 5, 5, 5, 5, 5, 5 }; + #endregion + + static sbyte[][] nextState = new sbyte[][] { + new sbyte[] {0, 0, 0, 0, 0, 0}, + new sbyte[] {-1, -1, 10, -1, -1, -1}, + new sbyte[] {-1, -1, -1, -1, -1, -1}, + new sbyte[] {-1, -1, 8, -1, -1, -1}, + new sbyte[] {-1, -1, 5, -1, -1, -1}, + new sbyte[] {-1, -1, 6, -1, -1, -1}, + new sbyte[] {-1, -1, 7, -1, -1, -1}, + null, + new sbyte[] {-1, -1, 9, -1, -1, -1}, + null, + null, + new sbyte[] {-1, 1, 2, 3, 4, 2} + }; + + + [SuppressMessage("Microsoft.Performance", "CA1810:InitializeReferenceTypeStaticFieldsInline")] + // Reason for suppression: cannot have self-reference in array initializer. + static Guesser() + { + nextState[7] = nextState[2]; + nextState[9] = nextState[2]; + nextState[10] = nextState[2]; + } + + int NextState() + { + if (code == ScanBuff.EndOfFile) + return eofNum; + else + return nextState[state][map[code]]; + } + #endregion + + public Guesser(System.IO.Stream file) { SetSource(file); } + + public void SetSource(System.IO.Stream source) + { + this.buffer = new BuildBuffer(source); + code = buffer.Read(); + } + + int Scan() + { + for (; ; ) + { + int next; + state = currentStart; + while ((next = NextState()) == goStart) + code = buffer.Read(); + + state = next; + code = buffer.Read(); + + while ((next = NextState()) > eofNum) + { + state = next; + code = buffer.Read(); + } + if (state <= maxAccept) + { + #region ActionSwitch +#pragma warning disable 162 + switch (state) + { + case eofNum: + switch (currentStart) + { + case 11: + if (utfX == 0 && uppr == 0) return -1; /* raw ascii */ + else if (uppr * 10 > utfX) return 0; /* default code page */ + else return 65001; /* UTF-8 encoding */ + break; + } + return EndToken; + case 1: // Recognized '{Upper128}', Shortest string "\xC0" + case 2: // Recognized '{Upper128}', Shortest string "\x80" + case 3: // Recognized '{Upper128}', Shortest string "\xE0" + case 4: // Recognized '{Upper128}', Shortest string "\xF0" + uppr++; + break; + case 5: // Recognized '{Utf8pfx4}{Utf8cont}', Shortest string "\xF0\x80" + uppr += 2; + break; + case 6: // Recognized '{Utf8pfx4}{Utf8cont}{2}', Shortest string "\xF0\x80\x80" + uppr += 3; + break; + case 7: // Recognized '{Utf8pfx4}{Utf8cont}{3}', Shortest string "\xF0\x80\x80\x80" + utfX += 3; + break; + case 8: // Recognized '{Utf8pfx3}{Utf8cont}', Shortest string "\xE0\x80" + uppr += 2; + break; + case 9: // Recognized '{Utf8pfx3}{Utf8cont}{2}', Shortest string "\xE0\x80\x80" + utfX += 2; + break; + case 10: // Recognized '{Utf8pfx2}{Utf8cont}', Shortest string "\xC0\x80" + utfX++; + break; + default: + break; + } +#pragma warning restore 162 + #endregion + } + } + } + } // end class Guesser + +#endif // !BYTEMODE +#endregion +#endif // !NOFILES + +// End of code copied from embedded resource +} diff --git a/src/DataStreamSuite/Bts.ColumnExpressions/Bts.ColumnExpressions.csproj b/src/DataStreamSuite/Bts.ColumnExpressions/Bts.ColumnExpressions.csproj index 39adf039f..3c12da7d5 100644 --- a/src/DataStreamSuite/Bts.ColumnExpressions/Bts.ColumnExpressions.csproj +++ b/src/DataStreamSuite/Bts.ColumnExpressions/Bts.ColumnExpressions.csproj @@ -1,7 +1,7 @@  - net8.0 + net7.0 enable enable diff --git a/src/DataStreamSuite/Bts.ColumnExpressions/ColumnExpression.cs b/src/DataStreamSuite/Bts.ColumnExpressions/ColumnExpression.cs index ee6a94217..e8c7abced 100644 --- a/src/DataStreamSuite/Bts.ColumnExpressions/ColumnExpression.cs +++ b/src/DataStreamSuite/Bts.ColumnExpressions/ColumnExpression.cs @@ -1,15 +1,21 @@ namespace Bts.ColumnExpressions; -public class ColumnExpression(string name, Type? type = null, object? value = null) { - public string Name { get; } = name; +public class ColumnExpression { + public ColumnExpression(string name, Type? type = null, object? value = null) { + Name = name; + Type = type + ?? value?.GetType() + ?? throw new ArgumentException( + "Type must be specified explicitly when value is null", nameof(type)); + Value = value + ?? DBNull.Value; + } - public Type Type { get; } = type - ?? value?.GetType() - ?? throw new ArgumentException( - "Type must be specified explicitly when value is null", nameof(type)); + public string Name { get; } - public object Value { get; } = value - ?? DBNull.Value; + public Type Type { get; } + + public object Value { get; } public static ColumnExpression Create(string name, T value, Type? type = null) { // Try to convert value to 'type' to verify they are compatible diff --git a/src/DataStreamSuite/Bts.ColumnExpressions/ColumnExpressionGrammar.y b/src/DataStreamSuite/Bts.ColumnExpressions/ColumnExpressionGrammar.y index f8c545933..942a62b0e 100644 --- a/src/DataStreamSuite/Bts.ColumnExpressions/ColumnExpressionGrammar.y +++ b/src/DataStreamSuite/Bts.ColumnExpressions/ColumnExpressionGrammar.y @@ -29,7 +29,7 @@ %% ColumnExpressions : - ColumnExpression { $$ = [$1]; } + ColumnExpression { $$ = ImmutableList.CreateRange(new[] { $1 }); } | ColumnExpressions ColumnExpression { $$ = $1.Add($2); } ; diff --git a/src/DataStreamSuite/Bts.ColumnSortExpressions/Bts.ColumnSortExpressions.csproj b/src/DataStreamSuite/Bts.ColumnSortExpressions/Bts.ColumnSortExpressions.csproj index 0702bb4d2..b8d537b2f 100644 --- a/src/DataStreamSuite/Bts.ColumnSortExpressions/Bts.ColumnSortExpressions.csproj +++ b/src/DataStreamSuite/Bts.ColumnSortExpressions/Bts.ColumnSortExpressions.csproj @@ -1,7 +1,7 @@  - net8.0 + net7.0 enable enable diff --git a/src/DataStreamSuite/Bts.ColumnSortExpressions/ColumnSortExpression.cs b/src/DataStreamSuite/Bts.ColumnSortExpressions/ColumnSortExpression.cs index fdb4da2fe..b52f0092c 100644 --- a/src/DataStreamSuite/Bts.ColumnSortExpressions/ColumnSortExpression.cs +++ b/src/DataStreamSuite/Bts.ColumnSortExpressions/ColumnSortExpression.cs @@ -1,6 +1,11 @@ namespace Bts.ColumnSortExpressions; -public class ColumnSortExpression(string name, bool isAscending = true) { - public string Name { get; } = name; - public bool IsAscending { get; } = isAscending; +public class ColumnSortExpression { + public ColumnSortExpression(string name, bool isAscending = true) { + Name = name; + IsAscending = isAscending; + } + + public string Name { get; } + public bool IsAscending { get; } } diff --git a/src/DataStreamSuite/Bts.ColumnSortExpressions/ColumnSortExpressionGrammar.y b/src/DataStreamSuite/Bts.ColumnSortExpressions/ColumnSortExpressionGrammar.y index 69413d975..e1f38872c 100644 --- a/src/DataStreamSuite/Bts.ColumnSortExpressions/ColumnSortExpressionGrammar.y +++ b/src/DataStreamSuite/Bts.ColumnSortExpressions/ColumnSortExpressionGrammar.y @@ -18,7 +18,7 @@ %% Columns : - Column { $$ = [$1]; } + Column { $$ = ImmutableList.CreateRange(new[] { $1 }); } | Columns Column { $$ = $1.Add($2); } ; diff --git a/src/DataStreamSuite/ColumnMapExpressions/GplexBuffers.cs b/src/DataStreamSuite/ColumnMapExpressions/GplexBuffers.cs index 76dd35443..1bcc08e09 100644 --- a/src/DataStreamSuite/ColumnMapExpressions/GplexBuffers.cs +++ b/src/DataStreamSuite/ColumnMapExpressions/GplexBuffers.cs @@ -1,720 +1,720 @@ -// ============================================================== -// -// This code automatically produced from an embedded resource. -// Do not edit this file, or it will become incompatible with -// the specification from which it was generated. -// -// ============================================================== - -using System; -using System.IO; -using System.Text; -using System.Collections.Generic; -using System.Diagnostics.CodeAnalysis; -using System.Runtime.Serialization; -using System.Globalization; - -namespace QUT.GplexBuffers -{ -// Code copied from GPLEX embedded resource - [Serializable] - internal class BufferException : Exception - { - public BufferException() { } - public BufferException(string message) : base(message) { } - public BufferException(string message, Exception innerException) - : base(message, innerException) { } - protected BufferException(SerializationInfo info, StreamingContext context) - : base(info, context) { } - } - - internal abstract class ScanBuff - { - private string fileNm; - - public const int EndOfFile = -1; - public const int UnicodeReplacementChar = 0xFFFD; - - public bool IsFile { get { return (fileNm != null); } } - public string FileName { get { return fileNm; } set { fileNm = value; } } - - public abstract int Pos { get; set; } - public abstract int Read(); - public virtual void Mark() { } - - public abstract string GetString(int begin, int limit); - - public static ScanBuff GetBuffer(string source) - { - return new StringBuffer(source); - } - - public static ScanBuff GetBuffer(IList source) - { - return new LineBuffer(source); - } - -#if (!NOFILES) - public static ScanBuff GetBuffer(Stream source) - { - return new BuildBuffer(source); - } - -#if (!BYTEMODE) - public static ScanBuff GetBuffer(Stream source, int fallbackCodePage) - { - return new BuildBuffer(source, fallbackCodePage); - } -#endif // !BYTEMODE -#endif // !NOFILES - } - - #region Buffer classes - - // ============================================================== - // ===== Definitions for various ScanBuff derived classes ==== - // ============================================================== - // =============== String input ================ - // ============================================================== - - /// - /// This class reads characters from a single string as - /// required, for example, by Visual Studio language services - /// - sealed class StringBuffer : ScanBuff - { - string str; // input buffer - int bPos; // current position in buffer - int sLen; - - public StringBuffer(string source) - { - this.str = source; - this.sLen = source.Length; - this.FileName = null; - } - - public override int Read() - { - if (bPos < sLen) return str[bPos++]; - else if (bPos == sLen) { bPos++; return '\n'; } // one strike, see new line - else { bPos++; return EndOfFile; } // two strikes and you're out! - } - - public override string GetString(int begin, int limit) - { - // "limit" can be greater than sLen with the BABEL - // option set. Read returns a "virtual" EOL if - // an attempt is made to read past the end of the - // string buffer. Without the guard any attempt - // to fetch yytext for a token that includes the - // EOL will throw an index exception. - if (limit > sLen) limit = sLen; - if (limit <= begin) return ""; - else return str.Substring(begin, limit - begin); - } - - public override int Pos - { - get { return bPos; } - set { bPos = value; } - } - - public override string ToString() { return "StringBuffer"; } - } - - // ============================================================== - // The LineBuff class contributed by Nigel Horspool, - // nigelh@cs.uvic.cs - // ============================================================== - - sealed class LineBuffer : ScanBuff - { - IList line; // list of source lines from a file - int numLines; // number of strings in line list - string curLine; // current line in that list - int cLine; // index of current line in the list - int curLen; // length of current line - int curLineStart; // position of line start in whole file - int curLineEnd; // position of line end in whole file - int maxPos; // max position ever visited in whole file - int cPos; // ordinal number of code in source - - // Constructed from a list of strings, one per source line. - // The lines have had trailing '\n' characters removed. - public LineBuffer(IList lineList) - { - line = lineList; - numLines = line.Count; - cPos = curLineStart = 0; - curLine = (numLines > 0 ? line[0] : ""); - maxPos = curLineEnd = curLen = curLine.Length; - cLine = 1; - FileName = null; - } - - public override int Read() - { - if (cPos < curLineEnd) - return curLine[cPos++ - curLineStart]; - if (cPos++ == curLineEnd) - return '\n'; - if (cLine >= numLines) - return EndOfFile; - curLine = line[cLine]; - curLen = curLine.Length; - curLineStart = curLineEnd + 1; - curLineEnd = curLineStart + curLen; - if (curLineEnd > maxPos) - maxPos = curLineEnd; - cLine++; - return curLen > 0 ? curLine[0] : '\n'; - } - - // To speed up searches for the line containing a position - private int cachedPosition; - private int cachedIxdex; - private int cachedLineStart; - - // Given a position pos within the entire source, the results are - // ix -- the index of the containing line - // lstart -- the position of the first character on that line - private void findIndex(int pos, out int ix, out int lstart) - { - if (pos >= cachedPosition) - { - ix = cachedIxdex; lstart = cachedLineStart; - } - else - { - ix = lstart = 0; - } - while (ix < numLines) - { - int len = line[ix].Length + 1; - if (pos < lstart + len) break; - lstart += len; - ix++; - } - cachedPosition = pos; - cachedIxdex = ix; - cachedLineStart = lstart; - } - - public override string GetString(int begin, int limit) - { - if (begin >= maxPos || limit <= begin) return ""; - int endIx, begIx, endLineStart, begLineStart; - findIndex(begin, out begIx, out begLineStart); - int begCol = begin - begLineStart; - findIndex(limit, out endIx, out endLineStart); - int endCol = limit - endLineStart; - string s = line[begIx]; - if (begIx == endIx) - { - // the usual case, substring all on one line - return (endCol <= s.Length) ? - s.Substring(begCol, endCol - begCol) - : s.Substring(begCol) + "\n"; - } - // the string spans multiple lines, yuk! - StringBuilder sb = new StringBuilder(); - if (begCol < s.Length) - sb.Append(s.Substring(begCol)); - for (; ; ) - { - sb.Append("\n"); - s = line[++begIx]; - if (begIx >= endIx) break; - sb.Append(s); - } - if (endCol <= s.Length) - { - sb.Append(s.Substring(0, endCol)); - } - else - { - sb.Append(s); - sb.Append("\n"); - } - return sb.ToString(); - } - - public override int Pos - { - get { return cPos; } - set - { - cPos = value; - findIndex(cPos, out cLine, out curLineStart); - // cLine should be the *next* line after curLine. - curLine = (cLine < numLines ? line[cLine++] : ""); - curLineEnd = curLineStart + curLine.Length; - } - } - - public override string ToString() { return "LineBuffer"; } - } - -#if (!NOFILES) - // ============================================================== - // ===== class BuildBuff : for unicode text files ======== - // ============================================================== - - class BuildBuffer : ScanBuff - { - // Double buffer for char stream. - class BufferElement - { - StringBuilder bldr = new StringBuilder(); - StringBuilder next = new StringBuilder(); - int minIx; - int maxIx; - int brkIx; - bool appendToNext; - - internal BufferElement() { } - - internal int MaxIndex { get { return maxIx; } } - // internal int MinIndex { get { return minIx; } } - - internal char this[int index] - { - get - { - if (index < minIx || index >= maxIx) - throw new BufferException("Index was outside data buffer"); - else if (index < brkIx) - return bldr[index - minIx]; - else - return next[index - brkIx]; - } - } - - internal void Append(char[] block, int count) - { - maxIx += count; - if (appendToNext) - this.next.Append(block, 0, count); - else - { - this.bldr.Append(block, 0, count); - brkIx = maxIx; - appendToNext = true; - } - } - - internal string GetString(int start, int limit) - { - if (limit <= start) - return ""; - if (start >= minIx && limit <= maxIx) - if (limit < brkIx) // String entirely in bldr builder - return bldr.ToString(start - minIx, limit - start); - else if (start >= brkIx) // String entirely in next builder - return next.ToString(start - brkIx, limit - start); - else // Must do a string-concatenation - return - bldr.ToString(start - minIx, brkIx - start) + - next.ToString(0, limit - brkIx); - else - throw new BufferException("String was outside data buffer"); - } - - internal void Mark(int limit) - { - if (limit > brkIx + 16) // Rotate blocks - { - StringBuilder temp = bldr; - bldr = next; - next = temp; - next.Length = 0; - minIx = brkIx; - brkIx = maxIx; - } - } - } - - BufferElement data = new BufferElement(); - - int bPos; // Postion index in the StringBuilder - BlockReader NextBlk; // Delegate that serves char-arrays; - - private string EncodingName - { - get - { - StreamReader rdr = NextBlk.Target as StreamReader; - return (rdr == null ? "raw-bytes" : rdr.CurrentEncoding.BodyName); - } - } - - public BuildBuffer(Stream stream) - { - FileStream fStrm = (stream as FileStream); - if (fStrm != null) FileName = fStrm.Name; - NextBlk = BlockReaderFactory.Raw(stream); - } - -#if (!BYTEMODE) - public BuildBuffer(Stream stream, int fallbackCodePage) - { - FileStream fStrm = (stream as FileStream); - if (fStrm != null) FileName = fStrm.Name; - NextBlk = BlockReaderFactory.Get(stream, fallbackCodePage); - } -#endif - - /// - /// Marks a conservative lower bound for the buffer, - /// allowing space to be reclaimed. If an application - /// needs to call GetString at arbitrary past locations - /// in the input stream, Mark() is not called. - /// - public override void Mark() { data.Mark(bPos - 2); } - - public override int Pos - { - get { return bPos; } - set { bPos = value; } - } - - - /// - /// Read returns the ordinal number of the next char, or - /// EOF (-1) for an end of stream. Note that the next - /// code point may require *two* calls of Read(). - /// - /// - public override int Read() - { - // - // Characters at positions - // [data.offset, data.offset + data.bldr.Length) - // are available in data.bldr. - // - if (bPos < data.MaxIndex) - { - // ch0 cannot be EOF - return (int)data[bPos++]; - } - else // Read from underlying stream - { - // Experimental code, blocks of page size - char[] chrs = new char[4096]; - int count = NextBlk(chrs, 0, 4096); - if (count == 0) - return EndOfFile; - else - { - data.Append(chrs, count); - return (int)data[bPos++]; - } - } - } - - public override string GetString(int begin, int limit) - { - return data.GetString(begin, limit); - } - - public override string ToString() - { - return "StringBuilder buffer, encoding: " + this.EncodingName; - } - } - - // =============== End ScanBuff-derived classes ================== - - internal delegate int BlockReader(char[] block, int index, int number); - - // A delegate factory, serving up a delegate that - // reads a block of characters from the underlying - // encoded stream, via a StreamReader object. - // - internal static class BlockReaderFactory - { - public static BlockReader Raw(Stream stream) - { - return delegate(char[] block, int index, int number) - { - byte[] b = new byte[number]; - int count = stream.Read(b, 0, number); - int i = 0; - int j = index; - for (; i < count; i++, j++) - block[j] = (char)b[i]; - return count; - }; - } - -#if (!BYTEMODE) - public static BlockReader Get(Stream stream, int fallbackCodePage) - { - Encoding encoding; - int preamble = Preamble(stream); - - if (preamble != 0) // There is a valid BOM here! - encoding = Encoding.GetEncoding(preamble); - else if (fallbackCodePage == -1) // Fallback is "raw" bytes - return Raw(stream); - else if (fallbackCodePage != -2) // Anything but "guess" - encoding = Encoding.GetEncoding(fallbackCodePage); - else // This is the "guess" option - { - int guess = new Guesser(stream).GuessCodePage(); - stream.Seek(0, SeekOrigin.Begin); - if (guess == -1) // ==> this is a 7-bit file - encoding = Encoding.ASCII; - else if (guess == 65001) - encoding = Encoding.UTF8; - else // ==> use the machine default - encoding = Encoding.Default; - } - StreamReader reader = new StreamReader(stream, encoding); - return reader.Read; - } - - static int Preamble(Stream stream) - { - int b0 = stream.ReadByte(); - int b1 = stream.ReadByte(); - - if (b0 == 0xfe && b1 == 0xff) - return 1201; // UTF16BE - if (b0 == 0xff && b1 == 0xfe) - return 1200; // UTF16LE - - int b2 = stream.ReadByte(); - if (b0 == 0xef && b1 == 0xbb && b2 == 0xbf) - return 65001; // UTF8 - // - // There is no unicode preamble, so we - // return denoter for the machine default. - // - stream.Seek(0, SeekOrigin.Begin); - return 0; - } -#endif // !BYTEMODE - } -#endif // !NOFILES - #endregion Buffer classes - - // ============================================================== - // ============ class CodePageHandling ============= - // ============================================================== -#if (!NOFILES) - public static class CodePageHandling - { - public static int GetCodePage(string option) - { - string command = option.ToUpperInvariant(); - if (command.StartsWith("CodePage:", StringComparison.OrdinalIgnoreCase)) - command = command.Substring(9); - try - { - if (command.Equals("RAW")) - return -1; - else if (command.Equals("GUESS")) - return -2; - else if (command.Equals("DEFAULT")) - return 0; - else if (char.IsDigit(command[0])) - return int.Parse(command, CultureInfo.InvariantCulture); - else - { - Encoding enc = Encoding.GetEncoding(command); - return enc.CodePage; - } - } - catch (FormatException) - { - Console.Error.WriteLine( - "Invalid format \"{0}\", using machine default", option); - } - catch (ArgumentException) - { - Console.Error.WriteLine( - "Unknown code page \"{0}\", using machine default", option); - } - return 0; - } - } -#region guesser -#if (!BYTEMODE) - // ============================================================== - // ============ Encoding Guesser ============= - // ============================================================== - - /// - /// This class provides a simple finite state automaton that - /// scans the file looking for (1) valid UTF-8 byte patterns, - /// (2) bytes >= 0x80 which are not part of a UTF-8 sequence. - /// The method then guesses whether it is UTF-8 or maybe some - /// local machine default encoding. This works well for the - /// various Latin encodings. - /// - internal class Guesser - { - ScanBuff buffer; - - public int GuessCodePage() { return Scan(); } - - const int maxAccept = 10; - const int initial = 0; - const int eofNum = 0; - const int goStart = -1; - const int INITIAL = 0; - const int EndToken = 0; - - #region user code - /* - * Reads the bytes of a file to determine if it is - * UTF-8 or a single-byte code page file. - */ - public long utfX; - public long uppr; - #endregion user code - - int state; - int currentStart = startState[0]; - int code; - - #region ScannerTables - static int[] startState = new int[] { 11, 0 }; - - #region CharacterMap - static sbyte[] map = new sbyte[256] { -/* '\0' */ 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, -/* '\x10' */ 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, -/* '\x20' */ 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, -/* '0' */ 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, -/* '@' */ 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, -/* 'P' */ 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, -/* '`' */ 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, -/* 'p' */ 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, -/* '\x80' */ 2, 2, 2, 2, 2, 2, 2, 2, 2, 2, 2, 2, 2, 2, 2, 2, -/* '\x90' */ 2, 2, 2, 2, 2, 2, 2, 2, 2, 2, 2, 2, 2, 2, 2, 2, -/* '\xA0' */ 2, 2, 2, 2, 2, 2, 2, 2, 2, 2, 2, 2, 2, 2, 2, 2, -/* '\xB0' */ 2, 2, 2, 2, 2, 2, 2, 2, 2, 2, 2, 2, 2, 2, 2, 2, -/* '\xC0' */ 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, -/* '\xD0' */ 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, -/* '\xE0' */ 3, 3, 3, 3, 3, 3, 3, 3, 3, 3, 3, 3, 3, 3, 3, 3, -/* '\xF0' */ 4, 4, 4, 4, 4, 4, 4, 4, 5, 5, 5, 5, 5, 5, 5, 5 }; - #endregion - - static sbyte[][] nextState = new sbyte[][] { - new sbyte[] {0, 0, 0, 0, 0, 0}, - new sbyte[] {-1, -1, 10, -1, -1, -1}, - new sbyte[] {-1, -1, -1, -1, -1, -1}, - new sbyte[] {-1, -1, 8, -1, -1, -1}, - new sbyte[] {-1, -1, 5, -1, -1, -1}, - new sbyte[] {-1, -1, 6, -1, -1, -1}, - new sbyte[] {-1, -1, 7, -1, -1, -1}, - null, - new sbyte[] {-1, -1, 9, -1, -1, -1}, - null, - null, - new sbyte[] {-1, 1, 2, 3, 4, 2} - }; - - - [SuppressMessage("Microsoft.Performance", "CA1810:InitializeReferenceTypeStaticFieldsInline")] - // Reason for suppression: cannot have self-reference in array initializer. - static Guesser() - { - nextState[7] = nextState[2]; - nextState[9] = nextState[2]; - nextState[10] = nextState[2]; - } - - int NextState() - { - if (code == ScanBuff.EndOfFile) - return eofNum; - else - return nextState[state][map[code]]; - } - #endregion - - public Guesser(System.IO.Stream file) { SetSource(file); } - - public void SetSource(System.IO.Stream source) - { - this.buffer = new BuildBuffer(source); - code = buffer.Read(); - } - - int Scan() - { - for (; ; ) - { - int next; - state = currentStart; - while ((next = NextState()) == goStart) - code = buffer.Read(); - - state = next; - code = buffer.Read(); - - while ((next = NextState()) > eofNum) - { - state = next; - code = buffer.Read(); - } - if (state <= maxAccept) - { - #region ActionSwitch -#pragma warning disable 162 - switch (state) - { - case eofNum: - switch (currentStart) - { - case 11: - if (utfX == 0 && uppr == 0) return -1; /* raw ascii */ - else if (uppr * 10 > utfX) return 0; /* default code page */ - else return 65001; /* UTF-8 encoding */ - break; - } - return EndToken; - case 1: // Recognized '{Upper128}', Shortest string "\xC0" - case 2: // Recognized '{Upper128}', Shortest string "\x80" - case 3: // Recognized '{Upper128}', Shortest string "\xE0" - case 4: // Recognized '{Upper128}', Shortest string "\xF0" - uppr++; - break; - case 5: // Recognized '{Utf8pfx4}{Utf8cont}', Shortest string "\xF0\x80" - uppr += 2; - break; - case 6: // Recognized '{Utf8pfx4}{Utf8cont}{2}', Shortest string "\xF0\x80\x80" - uppr += 3; - break; - case 7: // Recognized '{Utf8pfx4}{Utf8cont}{3}', Shortest string "\xF0\x80\x80\x80" - utfX += 3; - break; - case 8: // Recognized '{Utf8pfx3}{Utf8cont}', Shortest string "\xE0\x80" - uppr += 2; - break; - case 9: // Recognized '{Utf8pfx3}{Utf8cont}{2}', Shortest string "\xE0\x80\x80" - utfX += 2; - break; - case 10: // Recognized '{Utf8pfx2}{Utf8cont}', Shortest string "\xC0\x80" - utfX++; - break; - default: - break; - } -#pragma warning restore 162 - #endregion - } - } - } - } // end class Guesser - -#endif // !BYTEMODE -#endregion -#endif // !NOFILES - -// End of code copied from embedded resource -} +// ============================================================== +// +// This code automatically produced from an embedded resource. +// Do not edit this file, or it will become incompatible with +// the specification from which it was generated. +// +// ============================================================== + +using System; +using System.IO; +using System.Text; +using System.Collections.Generic; +using System.Diagnostics.CodeAnalysis; +using System.Runtime.Serialization; +using System.Globalization; + +namespace QUT.GplexBuffers +{ +// Code copied from GPLEX embedded resource + [Serializable] + internal class BufferException : Exception + { + public BufferException() { } + public BufferException(string message) : base(message) { } + public BufferException(string message, Exception innerException) + : base(message, innerException) { } + protected BufferException(SerializationInfo info, StreamingContext context) + : base(info, context) { } + } + + internal abstract class ScanBuff + { + private string fileNm; + + public const int EndOfFile = -1; + public const int UnicodeReplacementChar = 0xFFFD; + + public bool IsFile { get { return (fileNm != null); } } + public string FileName { get { return fileNm; } set { fileNm = value; } } + + public abstract int Pos { get; set; } + public abstract int Read(); + public virtual void Mark() { } + + public abstract string GetString(int begin, int limit); + + public static ScanBuff GetBuffer(string source) + { + return new StringBuffer(source); + } + + public static ScanBuff GetBuffer(IList source) + { + return new LineBuffer(source); + } + +#if (!NOFILES) + public static ScanBuff GetBuffer(Stream source) + { + return new BuildBuffer(source); + } + +#if (!BYTEMODE) + public static ScanBuff GetBuffer(Stream source, int fallbackCodePage) + { + return new BuildBuffer(source, fallbackCodePage); + } +#endif // !BYTEMODE +#endif // !NOFILES + } + + #region Buffer classes + + // ============================================================== + // ===== Definitions for various ScanBuff derived classes ==== + // ============================================================== + // =============== String input ================ + // ============================================================== + + /// + /// This class reads characters from a single string as + /// required, for example, by Visual Studio language services + /// + sealed class StringBuffer : ScanBuff + { + string str; // input buffer + int bPos; // current position in buffer + int sLen; + + public StringBuffer(string source) + { + this.str = source; + this.sLen = source.Length; + this.FileName = null; + } + + public override int Read() + { + if (bPos < sLen) return str[bPos++]; + else if (bPos == sLen) { bPos++; return '\n'; } // one strike, see new line + else { bPos++; return EndOfFile; } // two strikes and you're out! + } + + public override string GetString(int begin, int limit) + { + // "limit" can be greater than sLen with the BABEL + // option set. Read returns a "virtual" EOL if + // an attempt is made to read past the end of the + // string buffer. Without the guard any attempt + // to fetch yytext for a token that includes the + // EOL will throw an index exception. + if (limit > sLen) limit = sLen; + if (limit <= begin) return ""; + else return str.Substring(begin, limit - begin); + } + + public override int Pos + { + get { return bPos; } + set { bPos = value; } + } + + public override string ToString() { return "StringBuffer"; } + } + + // ============================================================== + // The LineBuff class contributed by Nigel Horspool, + // nigelh@cs.uvic.cs + // ============================================================== + + sealed class LineBuffer : ScanBuff + { + IList line; // list of source lines from a file + int numLines; // number of strings in line list + string curLine; // current line in that list + int cLine; // index of current line in the list + int curLen; // length of current line + int curLineStart; // position of line start in whole file + int curLineEnd; // position of line end in whole file + int maxPos; // max position ever visited in whole file + int cPos; // ordinal number of code in source + + // Constructed from a list of strings, one per source line. + // The lines have had trailing '\n' characters removed. + public LineBuffer(IList lineList) + { + line = lineList; + numLines = line.Count; + cPos = curLineStart = 0; + curLine = (numLines > 0 ? line[0] : ""); + maxPos = curLineEnd = curLen = curLine.Length; + cLine = 1; + FileName = null; + } + + public override int Read() + { + if (cPos < curLineEnd) + return curLine[cPos++ - curLineStart]; + if (cPos++ == curLineEnd) + return '\n'; + if (cLine >= numLines) + return EndOfFile; + curLine = line[cLine]; + curLen = curLine.Length; + curLineStart = curLineEnd + 1; + curLineEnd = curLineStart + curLen; + if (curLineEnd > maxPos) + maxPos = curLineEnd; + cLine++; + return curLen > 0 ? curLine[0] : '\n'; + } + + // To speed up searches for the line containing a position + private int cachedPosition; + private int cachedIxdex; + private int cachedLineStart; + + // Given a position pos within the entire source, the results are + // ix -- the index of the containing line + // lstart -- the position of the first character on that line + private void findIndex(int pos, out int ix, out int lstart) + { + if (pos >= cachedPosition) + { + ix = cachedIxdex; lstart = cachedLineStart; + } + else + { + ix = lstart = 0; + } + while (ix < numLines) + { + int len = line[ix].Length + 1; + if (pos < lstart + len) break; + lstart += len; + ix++; + } + cachedPosition = pos; + cachedIxdex = ix; + cachedLineStart = lstart; + } + + public override string GetString(int begin, int limit) + { + if (begin >= maxPos || limit <= begin) return ""; + int endIx, begIx, endLineStart, begLineStart; + findIndex(begin, out begIx, out begLineStart); + int begCol = begin - begLineStart; + findIndex(limit, out endIx, out endLineStart); + int endCol = limit - endLineStart; + string s = line[begIx]; + if (begIx == endIx) + { + // the usual case, substring all on one line + return (endCol <= s.Length) ? + s.Substring(begCol, endCol - begCol) + : s.Substring(begCol) + "\n"; + } + // the string spans multiple lines, yuk! + StringBuilder sb = new StringBuilder(); + if (begCol < s.Length) + sb.Append(s.Substring(begCol)); + for (; ; ) + { + sb.Append("\n"); + s = line[++begIx]; + if (begIx >= endIx) break; + sb.Append(s); + } + if (endCol <= s.Length) + { + sb.Append(s.Substring(0, endCol)); + } + else + { + sb.Append(s); + sb.Append("\n"); + } + return sb.ToString(); + } + + public override int Pos + { + get { return cPos; } + set + { + cPos = value; + findIndex(cPos, out cLine, out curLineStart); + // cLine should be the *next* line after curLine. + curLine = (cLine < numLines ? line[cLine++] : ""); + curLineEnd = curLineStart + curLine.Length; + } + } + + public override string ToString() { return "LineBuffer"; } + } + +#if (!NOFILES) + // ============================================================== + // ===== class BuildBuff : for unicode text files ======== + // ============================================================== + + class BuildBuffer : ScanBuff + { + // Double buffer for char stream. + class BufferElement + { + StringBuilder bldr = new StringBuilder(); + StringBuilder next = new StringBuilder(); + int minIx; + int maxIx; + int brkIx; + bool appendToNext; + + internal BufferElement() { } + + internal int MaxIndex { get { return maxIx; } } + // internal int MinIndex { get { return minIx; } } + + internal char this[int index] + { + get + { + if (index < minIx || index >= maxIx) + throw new BufferException("Index was outside data buffer"); + else if (index < brkIx) + return bldr[index - minIx]; + else + return next[index - brkIx]; + } + } + + internal void Append(char[] block, int count) + { + maxIx += count; + if (appendToNext) + this.next.Append(block, 0, count); + else + { + this.bldr.Append(block, 0, count); + brkIx = maxIx; + appendToNext = true; + } + } + + internal string GetString(int start, int limit) + { + if (limit <= start) + return ""; + if (start >= minIx && limit <= maxIx) + if (limit < brkIx) // String entirely in bldr builder + return bldr.ToString(start - minIx, limit - start); + else if (start >= brkIx) // String entirely in next builder + return next.ToString(start - brkIx, limit - start); + else // Must do a string-concatenation + return + bldr.ToString(start - minIx, brkIx - start) + + next.ToString(0, limit - brkIx); + else + throw new BufferException("String was outside data buffer"); + } + + internal void Mark(int limit) + { + if (limit > brkIx + 16) // Rotate blocks + { + StringBuilder temp = bldr; + bldr = next; + next = temp; + next.Length = 0; + minIx = brkIx; + brkIx = maxIx; + } + } + } + + BufferElement data = new BufferElement(); + + int bPos; // Postion index in the StringBuilder + BlockReader NextBlk; // Delegate that serves char-arrays; + + private string EncodingName + { + get + { + StreamReader rdr = NextBlk.Target as StreamReader; + return (rdr == null ? "raw-bytes" : rdr.CurrentEncoding.BodyName); + } + } + + public BuildBuffer(Stream stream) + { + FileStream fStrm = (stream as FileStream); + if (fStrm != null) FileName = fStrm.Name; + NextBlk = BlockReaderFactory.Raw(stream); + } + +#if (!BYTEMODE) + public BuildBuffer(Stream stream, int fallbackCodePage) + { + FileStream fStrm = (stream as FileStream); + if (fStrm != null) FileName = fStrm.Name; + NextBlk = BlockReaderFactory.Get(stream, fallbackCodePage); + } +#endif + + /// + /// Marks a conservative lower bound for the buffer, + /// allowing space to be reclaimed. If an application + /// needs to call GetString at arbitrary past locations + /// in the input stream, Mark() is not called. + /// + public override void Mark() { data.Mark(bPos - 2); } + + public override int Pos + { + get { return bPos; } + set { bPos = value; } + } + + + /// + /// Read returns the ordinal number of the next char, or + /// EOF (-1) for an end of stream. Note that the next + /// code point may require *two* calls of Read(). + /// + /// + public override int Read() + { + // + // Characters at positions + // [data.offset, data.offset + data.bldr.Length) + // are available in data.bldr. + // + if (bPos < data.MaxIndex) + { + // ch0 cannot be EOF + return (int)data[bPos++]; + } + else // Read from underlying stream + { + // Experimental code, blocks of page size + char[] chrs = new char[4096]; + int count = NextBlk(chrs, 0, 4096); + if (count == 0) + return EndOfFile; + else + { + data.Append(chrs, count); + return (int)data[bPos++]; + } + } + } + + public override string GetString(int begin, int limit) + { + return data.GetString(begin, limit); + } + + public override string ToString() + { + return "StringBuilder buffer, encoding: " + this.EncodingName; + } + } + + // =============== End ScanBuff-derived classes ================== + + internal delegate int BlockReader(char[] block, int index, int number); + + // A delegate factory, serving up a delegate that + // reads a block of characters from the underlying + // encoded stream, via a StreamReader object. + // + internal static class BlockReaderFactory + { + public static BlockReader Raw(Stream stream) + { + return delegate(char[] block, int index, int number) + { + byte[] b = new byte[number]; + int count = stream.Read(b, 0, number); + int i = 0; + int j = index; + for (; i < count; i++, j++) + block[j] = (char)b[i]; + return count; + }; + } + +#if (!BYTEMODE) + public static BlockReader Get(Stream stream, int fallbackCodePage) + { + Encoding encoding; + int preamble = Preamble(stream); + + if (preamble != 0) // There is a valid BOM here! + encoding = Encoding.GetEncoding(preamble); + else if (fallbackCodePage == -1) // Fallback is "raw" bytes + return Raw(stream); + else if (fallbackCodePage != -2) // Anything but "guess" + encoding = Encoding.GetEncoding(fallbackCodePage); + else // This is the "guess" option + { + int guess = new Guesser(stream).GuessCodePage(); + stream.Seek(0, SeekOrigin.Begin); + if (guess == -1) // ==> this is a 7-bit file + encoding = Encoding.ASCII; + else if (guess == 65001) + encoding = Encoding.UTF8; + else // ==> use the machine default + encoding = Encoding.Default; + } + StreamReader reader = new StreamReader(stream, encoding); + return reader.Read; + } + + static int Preamble(Stream stream) + { + int b0 = stream.ReadByte(); + int b1 = stream.ReadByte(); + + if (b0 == 0xfe && b1 == 0xff) + return 1201; // UTF16BE + if (b0 == 0xff && b1 == 0xfe) + return 1200; // UTF16LE + + int b2 = stream.ReadByte(); + if (b0 == 0xef && b1 == 0xbb && b2 == 0xbf) + return 65001; // UTF8 + // + // There is no unicode preamble, so we + // return denoter for the machine default. + // + stream.Seek(0, SeekOrigin.Begin); + return 0; + } +#endif // !BYTEMODE + } +#endif // !NOFILES + #endregion Buffer classes + + // ============================================================== + // ============ class CodePageHandling ============= + // ============================================================== +#if (!NOFILES) + public static class CodePageHandling + { + public static int GetCodePage(string option) + { + string command = option.ToUpperInvariant(); + if (command.StartsWith("CodePage:", StringComparison.OrdinalIgnoreCase)) + command = command.Substring(9); + try + { + if (command.Equals("RAW")) + return -1; + else if (command.Equals("GUESS")) + return -2; + else if (command.Equals("DEFAULT")) + return 0; + else if (char.IsDigit(command[0])) + return int.Parse(command, CultureInfo.InvariantCulture); + else + { + Encoding enc = Encoding.GetEncoding(command); + return enc.CodePage; + } + } + catch (FormatException) + { + Console.Error.WriteLine( + "Invalid format \"{0}\", using machine default", option); + } + catch (ArgumentException) + { + Console.Error.WriteLine( + "Unknown code page \"{0}\", using machine default", option); + } + return 0; + } + } +#region guesser +#if (!BYTEMODE) + // ============================================================== + // ============ Encoding Guesser ============= + // ============================================================== + + /// + /// This class provides a simple finite state automaton that + /// scans the file looking for (1) valid UTF-8 byte patterns, + /// (2) bytes >= 0x80 which are not part of a UTF-8 sequence. + /// The method then guesses whether it is UTF-8 or maybe some + /// local machine default encoding. This works well for the + /// various Latin encodings. + /// + internal class Guesser + { + ScanBuff buffer; + + public int GuessCodePage() { return Scan(); } + + const int maxAccept = 10; + const int initial = 0; + const int eofNum = 0; + const int goStart = -1; + const int INITIAL = 0; + const int EndToken = 0; + + #region user code + /* + * Reads the bytes of a file to determine if it is + * UTF-8 or a single-byte code page file. + */ + public long utfX; + public long uppr; + #endregion user code + + int state; + int currentStart = startState[0]; + int code; + + #region ScannerTables + static int[] startState = new int[] { 11, 0 }; + + #region CharacterMap + static sbyte[] map = new sbyte[256] { +/* '\0' */ 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, +/* '\x10' */ 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, +/* '\x20' */ 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, +/* '0' */ 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, +/* '@' */ 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, +/* 'P' */ 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, +/* '`' */ 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, +/* 'p' */ 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, +/* '\x80' */ 2, 2, 2, 2, 2, 2, 2, 2, 2, 2, 2, 2, 2, 2, 2, 2, +/* '\x90' */ 2, 2, 2, 2, 2, 2, 2, 2, 2, 2, 2, 2, 2, 2, 2, 2, +/* '\xA0' */ 2, 2, 2, 2, 2, 2, 2, 2, 2, 2, 2, 2, 2, 2, 2, 2, +/* '\xB0' */ 2, 2, 2, 2, 2, 2, 2, 2, 2, 2, 2, 2, 2, 2, 2, 2, +/* '\xC0' */ 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, +/* '\xD0' */ 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, +/* '\xE0' */ 3, 3, 3, 3, 3, 3, 3, 3, 3, 3, 3, 3, 3, 3, 3, 3, +/* '\xF0' */ 4, 4, 4, 4, 4, 4, 4, 4, 5, 5, 5, 5, 5, 5, 5, 5 }; + #endregion + + static sbyte[][] nextState = new sbyte[][] { + new sbyte[] {0, 0, 0, 0, 0, 0}, + new sbyte[] {-1, -1, 10, -1, -1, -1}, + new sbyte[] {-1, -1, -1, -1, -1, -1}, + new sbyte[] {-1, -1, 8, -1, -1, -1}, + new sbyte[] {-1, -1, 5, -1, -1, -1}, + new sbyte[] {-1, -1, 6, -1, -1, -1}, + new sbyte[] {-1, -1, 7, -1, -1, -1}, + null, + new sbyte[] {-1, -1, 9, -1, -1, -1}, + null, + null, + new sbyte[] {-1, 1, 2, 3, 4, 2} + }; + + + [SuppressMessage("Microsoft.Performance", "CA1810:InitializeReferenceTypeStaticFieldsInline")] + // Reason for suppression: cannot have self-reference in array initializer. + static Guesser() + { + nextState[7] = nextState[2]; + nextState[9] = nextState[2]; + nextState[10] = nextState[2]; + } + + int NextState() + { + if (code == ScanBuff.EndOfFile) + return eofNum; + else + return nextState[state][map[code]]; + } + #endregion + + public Guesser(System.IO.Stream file) { SetSource(file); } + + public void SetSource(System.IO.Stream source) + { + this.buffer = new BuildBuffer(source); + code = buffer.Read(); + } + + int Scan() + { + for (; ; ) + { + int next; + state = currentStart; + while ((next = NextState()) == goStart) + code = buffer.Read(); + + state = next; + code = buffer.Read(); + + while ((next = NextState()) > eofNum) + { + state = next; + code = buffer.Read(); + } + if (state <= maxAccept) + { + #region ActionSwitch +#pragma warning disable 162 + switch (state) + { + case eofNum: + switch (currentStart) + { + case 11: + if (utfX == 0 && uppr == 0) return -1; /* raw ascii */ + else if (uppr * 10 > utfX) return 0; /* default code page */ + else return 65001; /* UTF-8 encoding */ + break; + } + return EndToken; + case 1: // Recognized '{Upper128}', Shortest string "\xC0" + case 2: // Recognized '{Upper128}', Shortest string "\x80" + case 3: // Recognized '{Upper128}', Shortest string "\xE0" + case 4: // Recognized '{Upper128}', Shortest string "\xF0" + uppr++; + break; + case 5: // Recognized '{Utf8pfx4}{Utf8cont}', Shortest string "\xF0\x80" + uppr += 2; + break; + case 6: // Recognized '{Utf8pfx4}{Utf8cont}{2}', Shortest string "\xF0\x80\x80" + uppr += 3; + break; + case 7: // Recognized '{Utf8pfx4}{Utf8cont}{3}', Shortest string "\xF0\x80\x80\x80" + utfX += 3; + break; + case 8: // Recognized '{Utf8pfx3}{Utf8cont}', Shortest string "\xE0\x80" + uppr += 2; + break; + case 9: // Recognized '{Utf8pfx3}{Utf8cont}{2}', Shortest string "\xE0\x80\x80" + utfX += 2; + break; + case 10: // Recognized '{Utf8pfx2}{Utf8cont}', Shortest string "\xC0\x80" + utfX++; + break; + default: + break; + } +#pragma warning restore 162 + #endregion + } + } + } + } // end class Guesser + +#endif // !BYTEMODE +#endregion +#endif // !NOFILES + +// End of code copied from embedded resource +} diff --git a/src/DataStreamSuite/DataStream/Misc/PandasStream.cs b/src/DataStreamSuite/DataStream/Misc/PandasStream.cs new file mode 100644 index 000000000..a332a787a --- /dev/null +++ b/src/DataStreamSuite/DataStream/Misc/PandasStream.cs @@ -0,0 +1,62 @@ +// using System; +// using System.Diagnostics; + +// using BCG; +// using BCG.Utilities.General; + + +// namespace bts { + +// public class PandasStream : ViewStream, IWorker { +// //C:\Users\Owner\AppData\Local\Microsoft\WindowsApps\python.exe + +// public PandasStream(ViewStream vs, ViewStreamDatabase vsd, string pandas_code) { + +// } + +// public PandasStream(ViewStream vs, string pandas_code) : +// this(vs, ViewStreamDatabase.instance, pandas_code) { +// } + + +// protected override void myOnFirstSubscriber() { + +// // string python_code = @" +// // import winsound +// // frequency = 2500 # Set frequency (in Hz) +// // duration = 1000 # Set duration (in milliseconds) +// // winsound.Beep(frequency, duration) +// // "; +// string python_code = @" +// file_path = \"C:\\Temp\\test2.txt\\" # r prefix for raw string to avoid escape characters +// with open(file_path, \"w\") as f: +// f.write(\"Hello, world!\n\") +// "; + +// execute_python_code(python_code); +// } + +// static void execute_python_code(string python_code) { +// ProcessStartInfo start = new ProcessStartInfo(); +// start.FileName = "python"; // Python executable +// start.Arguments = "-c \"" + python_code.Replace("\"", "\\\"") + "\""; // Pass code as argument +// start.UseShellExecute = false; +// start.RedirectStandardOutput = true; + +// using (Process process = Process.Start(start)) { +// using (System.IO.StreamReader reader = process.StandardOutput) { +// string result = reader.ReadToEnd(); +// Console.Write(result); +// } +// } +// } + +// protected override void myOnLastSubscriber() { +// } + +// public void doWork() { + +// } + +// } +// } \ No newline at end of file diff --git a/src/DataStreamSuite/DataStream/Misc/test.py b/src/DataStreamSuite/DataStream/Misc/test.py new file mode 100644 index 000000000..01ae3d5f3 --- /dev/null +++ b/src/DataStreamSuite/DataStream/Misc/test.py @@ -0,0 +1,3 @@ +file_path = r"C:\Temp\test2.txt" # r prefix for raw string to avoid escape characters +with open(file_path, "w") as f: + f.write("Hello, world!\n") \ No newline at end of file diff --git a/src/DataStreamSuite/DataStream/Primitives/AddColumns/AddColumnsStream.cs b/src/DataStreamSuite/DataStream/Primitives/AddColumns/AddColumnsStream.cs index 0c4cec642..3c79de595 100644 --- a/src/DataStreamSuite/DataStream/Primitives/AddColumns/AddColumnsStream.cs +++ b/src/DataStreamSuite/DataStream/Primitives/AddColumns/AddColumnsStream.cs @@ -1,4 +1,6 @@ namespace BCG; -public class AddColumnsStream(ViewStream vs, string columnDef) - : MapStream(vs, new ColumnAdder(columnDef)) { } +public class AddColumnsStream : MapStream { + public AddColumnsStream(ViewStream vs, string columnDef) + : base(vs, new ColumnAdder(columnDef)) { } +} diff --git a/src/DataStreamSuite/DataStream/Primitives/AddColumns/ColumnAdder.cs b/src/DataStreamSuite/DataStream/Primitives/AddColumns/ColumnAdder.cs index 35ee9934a..75346aac8 100644 --- a/src/DataStreamSuite/DataStream/Primitives/AddColumns/ColumnAdder.cs +++ b/src/DataStreamSuite/DataStream/Primitives/AddColumns/ColumnAdder.cs @@ -13,14 +13,16 @@ namespace BCG; /// /// String represting the columns to add to the output table /// -public class ColumnAdder(string columnsExpression) : IStreamMapper { +public class ColumnAdder : IStreamMapper { // Store tuples of (column, index) to preserve order - protected readonly IDictionary myNewColumns = - (ColumnExpressionInterpreter.Evaluate(columnsExpression) ?? + protected readonly IDictionary myNewColumns; + protected IBlueTableSchema? mySchema; + + public ColumnAdder(string columnsExpression) => + myNewColumns = (ColumnExpressionInterpreter.Evaluate(columnsExpression) ?? throw new Exception("Invalid columnn expression: " + columnsExpression)) .Select((c, i) => (c, i)) .ToDictionary(c => c.c.Name, c => (c.c, c.i)); - protected IBlueTableSchema? mySchema; [MemberNotNull(nameof(mySchema))] public virtual IBlueRow map(IBlueRow row) { diff --git a/src/DataStreamSuite/DataStream/Primitives/GroupBy/MaxStream.cs b/src/DataStreamSuite/DataStream/Primitives/GroupBy/MaxStream.cs index 9f3b44fc6..8f55cab92 100644 --- a/src/DataStreamSuite/DataStream/Primitives/GroupBy/MaxStream.cs +++ b/src/DataStreamSuite/DataStream/Primitives/GroupBy/MaxStream.cs @@ -9,7 +9,7 @@ namespace BCG { new MaxGroupByEvaluator(outputCol, inputCols))); } - // public MaxStream(ViewStream stream, string outputCol) - // : this(new AddColumnsStream(stream, "fakeCol=0"), outputCol, new string[] { "fakeCol" }) { } + public MaxStream(ViewStream stream, string outputCol) + : this(new AddColumnsStream(stream, "fakeCol=0"), outputCol, new string[] { "fakeCol" }) { } } } diff --git a/src/DataStreamSuite/DataStream/Primitives/GroupBy/MinStream.cs b/src/DataStreamSuite/DataStream/Primitives/GroupBy/MinStream.cs index bb3d47495..02eb521af 100644 --- a/src/DataStreamSuite/DataStream/Primitives/GroupBy/MinStream.cs +++ b/src/DataStreamSuite/DataStream/Primitives/GroupBy/MinStream.cs @@ -7,7 +7,7 @@ namespace BCG { new MinGroupByEvaluator(outputCol, inputCol))); } - // public MinStream(ViewStream stream, string outputCol) - // : this(new AddColumnsStream(stream, "fakeCol=0"), outputCol, new string[] { "fakeCol" }) { } + public MinStream(ViewStream stream, string outputCol) + : this(new AddColumnsStream(stream, "fakeCol=0"), outputCol, new string[] { "fakeCol" }) { } } } diff --git a/src/DataStreamSuite/DataStream/Primitives/GroupBy/SumStream.cs b/src/DataStreamSuite/DataStream/Primitives/GroupBy/SumStream.cs index dced8d6f5..8ef26e25a 100644 --- a/src/DataStreamSuite/DataStream/Primitives/GroupBy/SumStream.cs +++ b/src/DataStreamSuite/DataStream/Primitives/GroupBy/SumStream.cs @@ -31,12 +31,12 @@ namespace BCG { new MultiColumnSumGroupByEvaluator(spaceSeparatedOutputCols, spaceSeparatedGroupByCols))); } - // public SumStream(ViewStream stream, string[] output_columns) { - // ViewStream rs = new AddColumnsStream(stream, "fake_col=0"); - // rs = new SumStream(rs, output_columns, new string[] { "fake_col" }, false); - // rs = new ProjectStream(rs, "!fake_col"); - // mySetRealSource(rs); - // } + public SumStream(ViewStream stream, string[] output_columns) { + ViewStream rs = new AddColumnsStream(stream, "fake_col=0"); + rs = new SumStream(rs, output_columns, new string[] { "fake_col" }, false); + rs = new ProjectStream(rs, "!fake_col"); + mySetRealSource(rs); + } public SumStream(ViewStream stream, string space_seperated_output_columns, string space_seperated_groupby_columns, bool force_null_into_other_columns) { string[] output_columns = space_seperated_output_columns.Split(' '); @@ -62,9 +62,9 @@ namespace BCG { mySetRealSource(rs); } - // public SumStream(ViewStream stream, string outputCol) - // : this(new AddColumnsStream(stream, "fakeCol=0"), outputCol, "fakeCol") { - // } + public SumStream(ViewStream stream, string outputCol) + : this(new AddColumnsStream(stream, "fakeCol=0"), outputCol, "fakeCol") { + } } public class IntSumStream : ViewStream { diff --git a/src/DataStreamSuite/DataStream/Primitives/Sort/SortStream.cs b/src/DataStreamSuite/DataStream/Primitives/Sort/SortStream.cs index 7aaf8aff0..3e6926b44 100644 --- a/src/DataStreamSuite/DataStream/Primitives/Sort/SortStream.cs +++ b/src/DataStreamSuite/DataStream/Primitives/Sort/SortStream.cs @@ -1,156 +1,99 @@ -// using Bts.ColumnSortExpressions; - -// using System.Collections; -// using System.Collections.Generic; -// using System.Diagnostics.CodeAnalysis; - -// namespace BCG; - -// /// -// /// Defines a Sort stream, with performance of O(n) for all the increments. It uses simple merge sort -// /// to merge the additions to the accumulates table -// /// -// public class SortStream : ViewStream { -// ViewStream stream; -// IList sortCols; -// IComparer? comparer; -// DataPacket? lastFiredPacket; -// DataPacket? lastReceivedPacket; - -// public SortStream(ViewStream stream, string sortExpression) { -// this.stream = stream; -// SortExpression = sortExpression; -// } - -// public SortStream(ViewStream vs, string colName, bool isAsc) { -// stream = vs; -// SortExpression = isAsc ? "+" + colName : "-" + colName; -// } - -// public SortStream(ViewStream stream, IComparer comparer) { -// this.stream = stream; -// this.comparer = comparer; -// sortCols = []; -// } - -// [MemberNotNull(nameof(sortCols))] -// public string SortExpression { -// set { -// comparer = null; -// lastFiredPacket = null; -// sortCols = ColumnSortExpressionInterpreter.Evaluate(value); -// if (lastReceivedPacket is not null) { -// myChangeHandler(lastReceivedPacket); -// } -// } -// } - -// public IComparer Comparer { -// set { -// comparer = value; -// lastFiredPacket = null; -// if (lastReceivedPacket is not null) { -// myChangeHandler(lastReceivedPacket); -// } -// } -// } - -// public void myChangeHandlerTest(object o) => -// myChangeHandler(o); - -// [MemberNotNull(nameof(lastFiredPacket))] -// internal void myChangeHandler(object o) { -// lastReceivedPacket = (DataPacket)o; -// // First callback, so we need to sort the whole table, this is the -// // only time we will do that for all the other sorting operations would -// // be sorting the increments and inserting them into the whole list -// if (lastFiredPacket is null) { -// lastFiredPacket = new() { -// newRows = comparer is null ? -// new SortedTable(lastReceivedPacket.newRows, sortCols) : -// new SortedTable(lastReceivedPacket.newRows, comparer) -// }; -// lastFiredPacket.addedRows = new OrderedBlueTable(lastFiredPacket.newRows); -// lastFiredPacket.deletedRows = new BlueTable(lastFiredPacket.newRows.schema); -// myFireEvent(lastFiredPacket.Clone()); -// return; -// } -// // Deletes -// lastFiredPacket.deletedRows = -// comparer == null ? -// new SortedTable(lastReceivedPacket.deletedRows, sortCols) : -// new SortedTable(lastReceivedPacket.deletedRows, comparer); -// ((SortedTable)lastFiredPacket.newRows).deleteRows(lastFiredPacket.deletedRows); -// // Adds -// lastFiredPacket.addedRows = new BlueTable(lastReceivedPacket.addedRows); -// ((SortedTable)lastFiredPacket.newRows).addRows(lastFiredPacket.addedRows); -// myFireEvent(lastFiredPacket.Clone()); -// } - -// protected override void myOnFirstSubscriber() => -// stream.onChange += myChangeHandler; - -// protected override void myOnLastSubscriber() { -// lastFiredPacket = null; -// lastReceivedPacket = null; -// stream.onChange -= myChangeHandler; -// } -// } - -using BCG; +using Bts.ColumnSortExpressions; +using System; using System.Collections; using System.Collections.Generic; -using System.Linq; +using System.Diagnostics.CodeAnalysis; +namespace BCG; -[DataStreamUserRespresentation(friendlyName: "sort")] -public class SortStreamTemp : ViewStream { - private ViewStream _vs; - private readonly string _sort_col_name; - private readonly bool _is_asc = true; - public SortStreamTemp(ViewStream vs, string sort_exp) { - _vs = vs; - if (sort_exp.StartsWith("+")) { - _is_asc = true; - _sort_col_name = sort_exp.Substring(1); - } else if (sort_exp.StartsWith("-")) { - _is_asc = false; - _sort_col_name = sort_exp.Substring(1); - } else { - _is_asc = true; - _sort_col_name = sort_exp; +/// +/// Defines a Sort stream, with performance of O(n) for all the increments. It uses simple merge sort +/// to merge the additions to the accumulates table +/// +public class SortStream : ViewStream { + ViewStream stream; + IList sortCols; + IComparer? comparer; + DataPacket? lastFiredPacket; + DataPacket? lastReceivedPacket; + + public SortStream(ViewStream stream, string sortExpression) { + this.stream = stream; + SortExpression = sortExpression; + } + + public SortStream(ViewStream vs, string colName, bool isAsc) { + stream = vs; + SortExpression = isAsc ? "+" + colName : "-" + colName; + } + + public SortStream(ViewStream stream, IComparer comparer) { + this.stream = stream; + this.comparer = comparer; + sortCols = Array.Empty(); + } + + [MemberNotNull(nameof(sortCols))] + public string SortExpression { + set { + comparer = null; + lastFiredPacket = null; + sortCols = ColumnSortExpressionInterpreter.Evaluate(value); + if (lastReceivedPacket is not null) { + myChangeHandler(lastReceivedPacket); + } } } - protected override void myOnFirstSubscriber() { - _vs.onChange += _handle_vs_update; + public IComparer Comparer { + set { + comparer = value; + lastFiredPacket = null; + if (lastReceivedPacket is not null) { + myChangeHandler(lastReceivedPacket); + } + } } + public void myChangeHandlerTest(object o) => + myChangeHandler(o); + + [MemberNotNull(nameof(lastFiredPacket))] + internal void myChangeHandler(object o) { + lastReceivedPacket = (DataPacket)o; + // First callback, so we need to sort the whole table, this is the + // only time we will do that for all the other sorting operations would + // be sorting the increments and inserting them into the whole list + if (lastFiredPacket is null) { + lastFiredPacket = new() { + newRows = comparer is null ? + new SortedTable(lastReceivedPacket.newRows, sortCols) : + new SortedTable(lastReceivedPacket.newRows, comparer) + }; + lastFiredPacket.addedRows = new OrderedBlueTable(lastFiredPacket.newRows); + lastFiredPacket.deletedRows = new BlueTable(lastFiredPacket.newRows.schema); + myFireEvent(lastFiredPacket.Clone()); + return; + } + // Deletes + lastFiredPacket.deletedRows = + comparer == null ? + new SortedTable(lastReceivedPacket.deletedRows, sortCols) : + new SortedTable(lastReceivedPacket.deletedRows, comparer); + ((SortedTable)lastFiredPacket.newRows).deleteRows(lastFiredPacket.deletedRows); + // Adds + lastFiredPacket.addedRows = new BlueTable(lastReceivedPacket.addedRows); + ((SortedTable)lastFiredPacket.newRows).addRows(lastFiredPacket.addedRows); + myFireEvent(lastFiredPacket.Clone()); + } + + protected override void myOnFirstSubscriber() => + stream.onChange += myChangeHandler; + protected override void myOnLastSubscriber() { - _vs.onChange -= _handle_vs_update; + lastFiredPacket = null; + lastReceivedPacket = null; + stream.onChange -= myChangeHandler; } - - private void _handle_vs_update(object dp_obj) { - var dp = dp_obj as DataPacket; - IBlueRowCollection nrows = dp.newRows; - var _out_schema = nrows.schema; - List row_list = new List(); - foreach (IBlueRow r in nrows) - row_list.Add(r); - //OrderBy sorts in ascedning order - var sorted_rows = row_list.OrderBy(r => r[_sort_col_name]).ToList(); - if (!_is_asc) - // reverse it to make it descending - sorted_rows.Reverse(); - //sort rows so we can do the diff - OrderedBlueTable rv = new OrderedBlueTable(_out_schema); - foreach (IBlueRow r in sorted_rows) { - rv.addRow(r); - } - - var dp_out = BlueDataUtility.build_dp_inc(new_rows: rv, myLastValue as DataPacket, true); - myFireEvent(dp_out); - - } -} \ No newline at end of file +} diff --git a/src/DataStreamSuite/DataStream/Primitives/Sort/SortedTable.cs b/src/DataStreamSuite/DataStream/Primitives/Sort/SortedTable.cs index 51252b16d..77dfecd42 100644 --- a/src/DataStreamSuite/DataStream/Primitives/Sort/SortedTable.cs +++ b/src/DataStreamSuite/DataStream/Primitives/Sort/SortedTable.cs @@ -11,9 +11,9 @@ namespace BCG; /// Represents an IBlueRowCollection with the rows sorted according to the /// public class SortedTable : IBlueRowCollection { - readonly List rows = []; - readonly Dictionary rowIndex = []; - readonly IList sortCols = []; + readonly List rows = new(0); + readonly Dictionary rowIndex = new(0); + readonly IList sortCols = Array.Empty(); readonly IComparer? comparer; /// diff --git a/src/DataStreamSuite/DataStream/Primitives/Top/TopStream.cs b/src/DataStreamSuite/DataStream/Primitives/Top/TopStream.cs index 10b68a4ad..c67dbd82c 100644 --- a/src/DataStreamSuite/DataStream/Primitives/Top/TopStream.cs +++ b/src/DataStreamSuite/DataStream/Primitives/Top/TopStream.cs @@ -1,49 +1,43 @@ -// using System; +using System; -// namespace BCG -// { -// public class TopStream : MethodDrivenStream -// { -// private int myTopCount; +namespace BCG { + public class TopStream : MethodDrivenStream { + private int myTopCount; -// public TopStream (ViewStream stream, int topCount) : base (stream) {myTopCount = topCount;} + public TopStream(ViewStream stream, int topCount) : base(stream) { myTopCount = topCount; } -// public TopStream (ViewStream stream, -// int topCount, -// string sortExpression) -// : -// this (sortExpression == null? -// stream : -// new SortStream (stream, sortExpression), topCount) -// {} + public TopStream(ViewStream stream, + int topCount, + string sortExpression) + : + this(sortExpression == null ? + stream : + new SortStream(stream, sortExpression), topCount) { } -// protected override void myOnTableClear () -// { -// myLastFiredPacket.deletedRows = myLastFiredPacket.newRows; -// myLastFiredPacket.newRows = new OrderedBlueTable (myLastFiredPacket.deletedRows.schema); -// myLastFiredPacket.addedRows = new OrderedBlueTable (myLastFiredPacket.deletedRows.schema); -// } + protected override void myOnTableClear() { + myLastFiredPacket.deletedRows = myLastFiredPacket.newRows; + myLastFiredPacket.newRows = new OrderedBlueTable(myLastFiredPacket.deletedRows.schema); + myLastFiredPacket.addedRows = new OrderedBlueTable(myLastFiredPacket.deletedRows.schema); + } -// protected override void myHandleFirstCall () -// { -// myLastFiredPacket = new DataPacket (); -// myLastFiredPacket.newRows = new OrderedBlueTable (myLastArrivedPacket.newRows, myTopCount); -// myLastFiredPacket.addedRows = new OrderedBlueTable(myLastFiredPacket.newRows); -// myLastFiredPacket.deletedRows = new OrderedBlueTable (myLastFiredPacket.newRows.schema); -// } + protected override void myHandleFirstCall() { + myLastFiredPacket = new DataPacket(); + myLastFiredPacket.newRows = new OrderedBlueTable(myLastArrivedPacket.newRows, myTopCount); + myLastFiredPacket.addedRows = new OrderedBlueTable(myLastFiredPacket.newRows); + myLastFiredPacket.deletedRows = new OrderedBlueTable(myLastFiredPacket.newRows.schema); + } -// protected override void myHandleIncrementalAdd () {} + protected override void myHandleIncrementalAdd() { } -// protected override void myHandleIncrementalDelete() -// { -// IBlueRowCollection oldRows = myLastFiredPacket.newRows; -// myLastFiredPacket.newRows = new OrderedBlueTable (myLastArrivedPacket.newRows, myTopCount); -// BlueDataUtility.computeDifferences (myLastFiredPacket.newRows, -// oldRows, -// myLastFiredPacket.schema.columnNames, -// out myLastFiredPacket.deletedRows, -// out myLastFiredPacket.addedRows); -// myFireSwitch = myLastFiredPacket.addedRows.Count + myLastFiredPacket.deletedRows.Count == 0 ? false : true; -// } -// } -// } \ No newline at end of file + protected override void myHandleIncrementalDelete() { + IBlueRowCollection oldRows = myLastFiredPacket.newRows; + myLastFiredPacket.newRows = new OrderedBlueTable(myLastArrivedPacket.newRows, myTopCount); + BlueDataUtility.computeDifferences(myLastFiredPacket.newRows, + oldRows, + myLastFiredPacket.schema.columnNames, + out myLastFiredPacket.deletedRows, + out myLastFiredPacket.addedRows); + myFireSwitch = myLastFiredPacket.addedRows.Count + myLastFiredPacket.deletedRows.Count == 0 ? false : true; + } + } +} \ No newline at end of file diff --git a/src/DataStreamSuite/DataStreamExpressions/GplexBuffers.cs b/src/DataStreamSuite/DataStreamExpressions/GplexBuffers.cs index 76dd35443..1bcc08e09 100644 --- a/src/DataStreamSuite/DataStreamExpressions/GplexBuffers.cs +++ b/src/DataStreamSuite/DataStreamExpressions/GplexBuffers.cs @@ -1,720 +1,720 @@ -// ============================================================== -// -// This code automatically produced from an embedded resource. -// Do not edit this file, or it will become incompatible with -// the specification from which it was generated. -// -// ============================================================== - -using System; -using System.IO; -using System.Text; -using System.Collections.Generic; -using System.Diagnostics.CodeAnalysis; -using System.Runtime.Serialization; -using System.Globalization; - -namespace QUT.GplexBuffers -{ -// Code copied from GPLEX embedded resource - [Serializable] - internal class BufferException : Exception - { - public BufferException() { } - public BufferException(string message) : base(message) { } - public BufferException(string message, Exception innerException) - : base(message, innerException) { } - protected BufferException(SerializationInfo info, StreamingContext context) - : base(info, context) { } - } - - internal abstract class ScanBuff - { - private string fileNm; - - public const int EndOfFile = -1; - public const int UnicodeReplacementChar = 0xFFFD; - - public bool IsFile { get { return (fileNm != null); } } - public string FileName { get { return fileNm; } set { fileNm = value; } } - - public abstract int Pos { get; set; } - public abstract int Read(); - public virtual void Mark() { } - - public abstract string GetString(int begin, int limit); - - public static ScanBuff GetBuffer(string source) - { - return new StringBuffer(source); - } - - public static ScanBuff GetBuffer(IList source) - { - return new LineBuffer(source); - } - -#if (!NOFILES) - public static ScanBuff GetBuffer(Stream source) - { - return new BuildBuffer(source); - } - -#if (!BYTEMODE) - public static ScanBuff GetBuffer(Stream source, int fallbackCodePage) - { - return new BuildBuffer(source, fallbackCodePage); - } -#endif // !BYTEMODE -#endif // !NOFILES - } - - #region Buffer classes - - // ============================================================== - // ===== Definitions for various ScanBuff derived classes ==== - // ============================================================== - // =============== String input ================ - // ============================================================== - - /// - /// This class reads characters from a single string as - /// required, for example, by Visual Studio language services - /// - sealed class StringBuffer : ScanBuff - { - string str; // input buffer - int bPos; // current position in buffer - int sLen; - - public StringBuffer(string source) - { - this.str = source; - this.sLen = source.Length; - this.FileName = null; - } - - public override int Read() - { - if (bPos < sLen) return str[bPos++]; - else if (bPos == sLen) { bPos++; return '\n'; } // one strike, see new line - else { bPos++; return EndOfFile; } // two strikes and you're out! - } - - public override string GetString(int begin, int limit) - { - // "limit" can be greater than sLen with the BABEL - // option set. Read returns a "virtual" EOL if - // an attempt is made to read past the end of the - // string buffer. Without the guard any attempt - // to fetch yytext for a token that includes the - // EOL will throw an index exception. - if (limit > sLen) limit = sLen; - if (limit <= begin) return ""; - else return str.Substring(begin, limit - begin); - } - - public override int Pos - { - get { return bPos; } - set { bPos = value; } - } - - public override string ToString() { return "StringBuffer"; } - } - - // ============================================================== - // The LineBuff class contributed by Nigel Horspool, - // nigelh@cs.uvic.cs - // ============================================================== - - sealed class LineBuffer : ScanBuff - { - IList line; // list of source lines from a file - int numLines; // number of strings in line list - string curLine; // current line in that list - int cLine; // index of current line in the list - int curLen; // length of current line - int curLineStart; // position of line start in whole file - int curLineEnd; // position of line end in whole file - int maxPos; // max position ever visited in whole file - int cPos; // ordinal number of code in source - - // Constructed from a list of strings, one per source line. - // The lines have had trailing '\n' characters removed. - public LineBuffer(IList lineList) - { - line = lineList; - numLines = line.Count; - cPos = curLineStart = 0; - curLine = (numLines > 0 ? line[0] : ""); - maxPos = curLineEnd = curLen = curLine.Length; - cLine = 1; - FileName = null; - } - - public override int Read() - { - if (cPos < curLineEnd) - return curLine[cPos++ - curLineStart]; - if (cPos++ == curLineEnd) - return '\n'; - if (cLine >= numLines) - return EndOfFile; - curLine = line[cLine]; - curLen = curLine.Length; - curLineStart = curLineEnd + 1; - curLineEnd = curLineStart + curLen; - if (curLineEnd > maxPos) - maxPos = curLineEnd; - cLine++; - return curLen > 0 ? curLine[0] : '\n'; - } - - // To speed up searches for the line containing a position - private int cachedPosition; - private int cachedIxdex; - private int cachedLineStart; - - // Given a position pos within the entire source, the results are - // ix -- the index of the containing line - // lstart -- the position of the first character on that line - private void findIndex(int pos, out int ix, out int lstart) - { - if (pos >= cachedPosition) - { - ix = cachedIxdex; lstart = cachedLineStart; - } - else - { - ix = lstart = 0; - } - while (ix < numLines) - { - int len = line[ix].Length + 1; - if (pos < lstart + len) break; - lstart += len; - ix++; - } - cachedPosition = pos; - cachedIxdex = ix; - cachedLineStart = lstart; - } - - public override string GetString(int begin, int limit) - { - if (begin >= maxPos || limit <= begin) return ""; - int endIx, begIx, endLineStart, begLineStart; - findIndex(begin, out begIx, out begLineStart); - int begCol = begin - begLineStart; - findIndex(limit, out endIx, out endLineStart); - int endCol = limit - endLineStart; - string s = line[begIx]; - if (begIx == endIx) - { - // the usual case, substring all on one line - return (endCol <= s.Length) ? - s.Substring(begCol, endCol - begCol) - : s.Substring(begCol) + "\n"; - } - // the string spans multiple lines, yuk! - StringBuilder sb = new StringBuilder(); - if (begCol < s.Length) - sb.Append(s.Substring(begCol)); - for (; ; ) - { - sb.Append("\n"); - s = line[++begIx]; - if (begIx >= endIx) break; - sb.Append(s); - } - if (endCol <= s.Length) - { - sb.Append(s.Substring(0, endCol)); - } - else - { - sb.Append(s); - sb.Append("\n"); - } - return sb.ToString(); - } - - public override int Pos - { - get { return cPos; } - set - { - cPos = value; - findIndex(cPos, out cLine, out curLineStart); - // cLine should be the *next* line after curLine. - curLine = (cLine < numLines ? line[cLine++] : ""); - curLineEnd = curLineStart + curLine.Length; - } - } - - public override string ToString() { return "LineBuffer"; } - } - -#if (!NOFILES) - // ============================================================== - // ===== class BuildBuff : for unicode text files ======== - // ============================================================== - - class BuildBuffer : ScanBuff - { - // Double buffer for char stream. - class BufferElement - { - StringBuilder bldr = new StringBuilder(); - StringBuilder next = new StringBuilder(); - int minIx; - int maxIx; - int brkIx; - bool appendToNext; - - internal BufferElement() { } - - internal int MaxIndex { get { return maxIx; } } - // internal int MinIndex { get { return minIx; } } - - internal char this[int index] - { - get - { - if (index < minIx || index >= maxIx) - throw new BufferException("Index was outside data buffer"); - else if (index < brkIx) - return bldr[index - minIx]; - else - return next[index - brkIx]; - } - } - - internal void Append(char[] block, int count) - { - maxIx += count; - if (appendToNext) - this.next.Append(block, 0, count); - else - { - this.bldr.Append(block, 0, count); - brkIx = maxIx; - appendToNext = true; - } - } - - internal string GetString(int start, int limit) - { - if (limit <= start) - return ""; - if (start >= minIx && limit <= maxIx) - if (limit < brkIx) // String entirely in bldr builder - return bldr.ToString(start - minIx, limit - start); - else if (start >= brkIx) // String entirely in next builder - return next.ToString(start - brkIx, limit - start); - else // Must do a string-concatenation - return - bldr.ToString(start - minIx, brkIx - start) + - next.ToString(0, limit - brkIx); - else - throw new BufferException("String was outside data buffer"); - } - - internal void Mark(int limit) - { - if (limit > brkIx + 16) // Rotate blocks - { - StringBuilder temp = bldr; - bldr = next; - next = temp; - next.Length = 0; - minIx = brkIx; - brkIx = maxIx; - } - } - } - - BufferElement data = new BufferElement(); - - int bPos; // Postion index in the StringBuilder - BlockReader NextBlk; // Delegate that serves char-arrays; - - private string EncodingName - { - get - { - StreamReader rdr = NextBlk.Target as StreamReader; - return (rdr == null ? "raw-bytes" : rdr.CurrentEncoding.BodyName); - } - } - - public BuildBuffer(Stream stream) - { - FileStream fStrm = (stream as FileStream); - if (fStrm != null) FileName = fStrm.Name; - NextBlk = BlockReaderFactory.Raw(stream); - } - -#if (!BYTEMODE) - public BuildBuffer(Stream stream, int fallbackCodePage) - { - FileStream fStrm = (stream as FileStream); - if (fStrm != null) FileName = fStrm.Name; - NextBlk = BlockReaderFactory.Get(stream, fallbackCodePage); - } -#endif - - /// - /// Marks a conservative lower bound for the buffer, - /// allowing space to be reclaimed. If an application - /// needs to call GetString at arbitrary past locations - /// in the input stream, Mark() is not called. - /// - public override void Mark() { data.Mark(bPos - 2); } - - public override int Pos - { - get { return bPos; } - set { bPos = value; } - } - - - /// - /// Read returns the ordinal number of the next char, or - /// EOF (-1) for an end of stream. Note that the next - /// code point may require *two* calls of Read(). - /// - /// - public override int Read() - { - // - // Characters at positions - // [data.offset, data.offset + data.bldr.Length) - // are available in data.bldr. - // - if (bPos < data.MaxIndex) - { - // ch0 cannot be EOF - return (int)data[bPos++]; - } - else // Read from underlying stream - { - // Experimental code, blocks of page size - char[] chrs = new char[4096]; - int count = NextBlk(chrs, 0, 4096); - if (count == 0) - return EndOfFile; - else - { - data.Append(chrs, count); - return (int)data[bPos++]; - } - } - } - - public override string GetString(int begin, int limit) - { - return data.GetString(begin, limit); - } - - public override string ToString() - { - return "StringBuilder buffer, encoding: " + this.EncodingName; - } - } - - // =============== End ScanBuff-derived classes ================== - - internal delegate int BlockReader(char[] block, int index, int number); - - // A delegate factory, serving up a delegate that - // reads a block of characters from the underlying - // encoded stream, via a StreamReader object. - // - internal static class BlockReaderFactory - { - public static BlockReader Raw(Stream stream) - { - return delegate(char[] block, int index, int number) - { - byte[] b = new byte[number]; - int count = stream.Read(b, 0, number); - int i = 0; - int j = index; - for (; i < count; i++, j++) - block[j] = (char)b[i]; - return count; - }; - } - -#if (!BYTEMODE) - public static BlockReader Get(Stream stream, int fallbackCodePage) - { - Encoding encoding; - int preamble = Preamble(stream); - - if (preamble != 0) // There is a valid BOM here! - encoding = Encoding.GetEncoding(preamble); - else if (fallbackCodePage == -1) // Fallback is "raw" bytes - return Raw(stream); - else if (fallbackCodePage != -2) // Anything but "guess" - encoding = Encoding.GetEncoding(fallbackCodePage); - else // This is the "guess" option - { - int guess = new Guesser(stream).GuessCodePage(); - stream.Seek(0, SeekOrigin.Begin); - if (guess == -1) // ==> this is a 7-bit file - encoding = Encoding.ASCII; - else if (guess == 65001) - encoding = Encoding.UTF8; - else // ==> use the machine default - encoding = Encoding.Default; - } - StreamReader reader = new StreamReader(stream, encoding); - return reader.Read; - } - - static int Preamble(Stream stream) - { - int b0 = stream.ReadByte(); - int b1 = stream.ReadByte(); - - if (b0 == 0xfe && b1 == 0xff) - return 1201; // UTF16BE - if (b0 == 0xff && b1 == 0xfe) - return 1200; // UTF16LE - - int b2 = stream.ReadByte(); - if (b0 == 0xef && b1 == 0xbb && b2 == 0xbf) - return 65001; // UTF8 - // - // There is no unicode preamble, so we - // return denoter for the machine default. - // - stream.Seek(0, SeekOrigin.Begin); - return 0; - } -#endif // !BYTEMODE - } -#endif // !NOFILES - #endregion Buffer classes - - // ============================================================== - // ============ class CodePageHandling ============= - // ============================================================== -#if (!NOFILES) - public static class CodePageHandling - { - public static int GetCodePage(string option) - { - string command = option.ToUpperInvariant(); - if (command.StartsWith("CodePage:", StringComparison.OrdinalIgnoreCase)) - command = command.Substring(9); - try - { - if (command.Equals("RAW")) - return -1; - else if (command.Equals("GUESS")) - return -2; - else if (command.Equals("DEFAULT")) - return 0; - else if (char.IsDigit(command[0])) - return int.Parse(command, CultureInfo.InvariantCulture); - else - { - Encoding enc = Encoding.GetEncoding(command); - return enc.CodePage; - } - } - catch (FormatException) - { - Console.Error.WriteLine( - "Invalid format \"{0}\", using machine default", option); - } - catch (ArgumentException) - { - Console.Error.WriteLine( - "Unknown code page \"{0}\", using machine default", option); - } - return 0; - } - } -#region guesser -#if (!BYTEMODE) - // ============================================================== - // ============ Encoding Guesser ============= - // ============================================================== - - /// - /// This class provides a simple finite state automaton that - /// scans the file looking for (1) valid UTF-8 byte patterns, - /// (2) bytes >= 0x80 which are not part of a UTF-8 sequence. - /// The method then guesses whether it is UTF-8 or maybe some - /// local machine default encoding. This works well for the - /// various Latin encodings. - /// - internal class Guesser - { - ScanBuff buffer; - - public int GuessCodePage() { return Scan(); } - - const int maxAccept = 10; - const int initial = 0; - const int eofNum = 0; - const int goStart = -1; - const int INITIAL = 0; - const int EndToken = 0; - - #region user code - /* - * Reads the bytes of a file to determine if it is - * UTF-8 or a single-byte code page file. - */ - public long utfX; - public long uppr; - #endregion user code - - int state; - int currentStart = startState[0]; - int code; - - #region ScannerTables - static int[] startState = new int[] { 11, 0 }; - - #region CharacterMap - static sbyte[] map = new sbyte[256] { -/* '\0' */ 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, -/* '\x10' */ 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, -/* '\x20' */ 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, -/* '0' */ 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, -/* '@' */ 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, -/* 'P' */ 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, -/* '`' */ 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, -/* 'p' */ 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, -/* '\x80' */ 2, 2, 2, 2, 2, 2, 2, 2, 2, 2, 2, 2, 2, 2, 2, 2, -/* '\x90' */ 2, 2, 2, 2, 2, 2, 2, 2, 2, 2, 2, 2, 2, 2, 2, 2, -/* '\xA0' */ 2, 2, 2, 2, 2, 2, 2, 2, 2, 2, 2, 2, 2, 2, 2, 2, -/* '\xB0' */ 2, 2, 2, 2, 2, 2, 2, 2, 2, 2, 2, 2, 2, 2, 2, 2, -/* '\xC0' */ 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, -/* '\xD0' */ 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, -/* '\xE0' */ 3, 3, 3, 3, 3, 3, 3, 3, 3, 3, 3, 3, 3, 3, 3, 3, -/* '\xF0' */ 4, 4, 4, 4, 4, 4, 4, 4, 5, 5, 5, 5, 5, 5, 5, 5 }; - #endregion - - static sbyte[][] nextState = new sbyte[][] { - new sbyte[] {0, 0, 0, 0, 0, 0}, - new sbyte[] {-1, -1, 10, -1, -1, -1}, - new sbyte[] {-1, -1, -1, -1, -1, -1}, - new sbyte[] {-1, -1, 8, -1, -1, -1}, - new sbyte[] {-1, -1, 5, -1, -1, -1}, - new sbyte[] {-1, -1, 6, -1, -1, -1}, - new sbyte[] {-1, -1, 7, -1, -1, -1}, - null, - new sbyte[] {-1, -1, 9, -1, -1, -1}, - null, - null, - new sbyte[] {-1, 1, 2, 3, 4, 2} - }; - - - [SuppressMessage("Microsoft.Performance", "CA1810:InitializeReferenceTypeStaticFieldsInline")] - // Reason for suppression: cannot have self-reference in array initializer. - static Guesser() - { - nextState[7] = nextState[2]; - nextState[9] = nextState[2]; - nextState[10] = nextState[2]; - } - - int NextState() - { - if (code == ScanBuff.EndOfFile) - return eofNum; - else - return nextState[state][map[code]]; - } - #endregion - - public Guesser(System.IO.Stream file) { SetSource(file); } - - public void SetSource(System.IO.Stream source) - { - this.buffer = new BuildBuffer(source); - code = buffer.Read(); - } - - int Scan() - { - for (; ; ) - { - int next; - state = currentStart; - while ((next = NextState()) == goStart) - code = buffer.Read(); - - state = next; - code = buffer.Read(); - - while ((next = NextState()) > eofNum) - { - state = next; - code = buffer.Read(); - } - if (state <= maxAccept) - { - #region ActionSwitch -#pragma warning disable 162 - switch (state) - { - case eofNum: - switch (currentStart) - { - case 11: - if (utfX == 0 && uppr == 0) return -1; /* raw ascii */ - else if (uppr * 10 > utfX) return 0; /* default code page */ - else return 65001; /* UTF-8 encoding */ - break; - } - return EndToken; - case 1: // Recognized '{Upper128}', Shortest string "\xC0" - case 2: // Recognized '{Upper128}', Shortest string "\x80" - case 3: // Recognized '{Upper128}', Shortest string "\xE0" - case 4: // Recognized '{Upper128}', Shortest string "\xF0" - uppr++; - break; - case 5: // Recognized '{Utf8pfx4}{Utf8cont}', Shortest string "\xF0\x80" - uppr += 2; - break; - case 6: // Recognized '{Utf8pfx4}{Utf8cont}{2}', Shortest string "\xF0\x80\x80" - uppr += 3; - break; - case 7: // Recognized '{Utf8pfx4}{Utf8cont}{3}', Shortest string "\xF0\x80\x80\x80" - utfX += 3; - break; - case 8: // Recognized '{Utf8pfx3}{Utf8cont}', Shortest string "\xE0\x80" - uppr += 2; - break; - case 9: // Recognized '{Utf8pfx3}{Utf8cont}{2}', Shortest string "\xE0\x80\x80" - utfX += 2; - break; - case 10: // Recognized '{Utf8pfx2}{Utf8cont}', Shortest string "\xC0\x80" - utfX++; - break; - default: - break; - } -#pragma warning restore 162 - #endregion - } - } - } - } // end class Guesser - -#endif // !BYTEMODE -#endregion -#endif // !NOFILES - -// End of code copied from embedded resource -} +// ============================================================== +// +// This code automatically produced from an embedded resource. +// Do not edit this file, or it will become incompatible with +// the specification from which it was generated. +// +// ============================================================== + +using System; +using System.IO; +using System.Text; +using System.Collections.Generic; +using System.Diagnostics.CodeAnalysis; +using System.Runtime.Serialization; +using System.Globalization; + +namespace QUT.GplexBuffers +{ +// Code copied from GPLEX embedded resource + [Serializable] + internal class BufferException : Exception + { + public BufferException() { } + public BufferException(string message) : base(message) { } + public BufferException(string message, Exception innerException) + : base(message, innerException) { } + protected BufferException(SerializationInfo info, StreamingContext context) + : base(info, context) { } + } + + internal abstract class ScanBuff + { + private string fileNm; + + public const int EndOfFile = -1; + public const int UnicodeReplacementChar = 0xFFFD; + + public bool IsFile { get { return (fileNm != null); } } + public string FileName { get { return fileNm; } set { fileNm = value; } } + + public abstract int Pos { get; set; } + public abstract int Read(); + public virtual void Mark() { } + + public abstract string GetString(int begin, int limit); + + public static ScanBuff GetBuffer(string source) + { + return new StringBuffer(source); + } + + public static ScanBuff GetBuffer(IList source) + { + return new LineBuffer(source); + } + +#if (!NOFILES) + public static ScanBuff GetBuffer(Stream source) + { + return new BuildBuffer(source); + } + +#if (!BYTEMODE) + public static ScanBuff GetBuffer(Stream source, int fallbackCodePage) + { + return new BuildBuffer(source, fallbackCodePage); + } +#endif // !BYTEMODE +#endif // !NOFILES + } + + #region Buffer classes + + // ============================================================== + // ===== Definitions for various ScanBuff derived classes ==== + // ============================================================== + // =============== String input ================ + // ============================================================== + + /// + /// This class reads characters from a single string as + /// required, for example, by Visual Studio language services + /// + sealed class StringBuffer : ScanBuff + { + string str; // input buffer + int bPos; // current position in buffer + int sLen; + + public StringBuffer(string source) + { + this.str = source; + this.sLen = source.Length; + this.FileName = null; + } + + public override int Read() + { + if (bPos < sLen) return str[bPos++]; + else if (bPos == sLen) { bPos++; return '\n'; } // one strike, see new line + else { bPos++; return EndOfFile; } // two strikes and you're out! + } + + public override string GetString(int begin, int limit) + { + // "limit" can be greater than sLen with the BABEL + // option set. Read returns a "virtual" EOL if + // an attempt is made to read past the end of the + // string buffer. Without the guard any attempt + // to fetch yytext for a token that includes the + // EOL will throw an index exception. + if (limit > sLen) limit = sLen; + if (limit <= begin) return ""; + else return str.Substring(begin, limit - begin); + } + + public override int Pos + { + get { return bPos; } + set { bPos = value; } + } + + public override string ToString() { return "StringBuffer"; } + } + + // ============================================================== + // The LineBuff class contributed by Nigel Horspool, + // nigelh@cs.uvic.cs + // ============================================================== + + sealed class LineBuffer : ScanBuff + { + IList line; // list of source lines from a file + int numLines; // number of strings in line list + string curLine; // current line in that list + int cLine; // index of current line in the list + int curLen; // length of current line + int curLineStart; // position of line start in whole file + int curLineEnd; // position of line end in whole file + int maxPos; // max position ever visited in whole file + int cPos; // ordinal number of code in source + + // Constructed from a list of strings, one per source line. + // The lines have had trailing '\n' characters removed. + public LineBuffer(IList lineList) + { + line = lineList; + numLines = line.Count; + cPos = curLineStart = 0; + curLine = (numLines > 0 ? line[0] : ""); + maxPos = curLineEnd = curLen = curLine.Length; + cLine = 1; + FileName = null; + } + + public override int Read() + { + if (cPos < curLineEnd) + return curLine[cPos++ - curLineStart]; + if (cPos++ == curLineEnd) + return '\n'; + if (cLine >= numLines) + return EndOfFile; + curLine = line[cLine]; + curLen = curLine.Length; + curLineStart = curLineEnd + 1; + curLineEnd = curLineStart + curLen; + if (curLineEnd > maxPos) + maxPos = curLineEnd; + cLine++; + return curLen > 0 ? curLine[0] : '\n'; + } + + // To speed up searches for the line containing a position + private int cachedPosition; + private int cachedIxdex; + private int cachedLineStart; + + // Given a position pos within the entire source, the results are + // ix -- the index of the containing line + // lstart -- the position of the first character on that line + private void findIndex(int pos, out int ix, out int lstart) + { + if (pos >= cachedPosition) + { + ix = cachedIxdex; lstart = cachedLineStart; + } + else + { + ix = lstart = 0; + } + while (ix < numLines) + { + int len = line[ix].Length + 1; + if (pos < lstart + len) break; + lstart += len; + ix++; + } + cachedPosition = pos; + cachedIxdex = ix; + cachedLineStart = lstart; + } + + public override string GetString(int begin, int limit) + { + if (begin >= maxPos || limit <= begin) return ""; + int endIx, begIx, endLineStart, begLineStart; + findIndex(begin, out begIx, out begLineStart); + int begCol = begin - begLineStart; + findIndex(limit, out endIx, out endLineStart); + int endCol = limit - endLineStart; + string s = line[begIx]; + if (begIx == endIx) + { + // the usual case, substring all on one line + return (endCol <= s.Length) ? + s.Substring(begCol, endCol - begCol) + : s.Substring(begCol) + "\n"; + } + // the string spans multiple lines, yuk! + StringBuilder sb = new StringBuilder(); + if (begCol < s.Length) + sb.Append(s.Substring(begCol)); + for (; ; ) + { + sb.Append("\n"); + s = line[++begIx]; + if (begIx >= endIx) break; + sb.Append(s); + } + if (endCol <= s.Length) + { + sb.Append(s.Substring(0, endCol)); + } + else + { + sb.Append(s); + sb.Append("\n"); + } + return sb.ToString(); + } + + public override int Pos + { + get { return cPos; } + set + { + cPos = value; + findIndex(cPos, out cLine, out curLineStart); + // cLine should be the *next* line after curLine. + curLine = (cLine < numLines ? line[cLine++] : ""); + curLineEnd = curLineStart + curLine.Length; + } + } + + public override string ToString() { return "LineBuffer"; } + } + +#if (!NOFILES) + // ============================================================== + // ===== class BuildBuff : for unicode text files ======== + // ============================================================== + + class BuildBuffer : ScanBuff + { + // Double buffer for char stream. + class BufferElement + { + StringBuilder bldr = new StringBuilder(); + StringBuilder next = new StringBuilder(); + int minIx; + int maxIx; + int brkIx; + bool appendToNext; + + internal BufferElement() { } + + internal int MaxIndex { get { return maxIx; } } + // internal int MinIndex { get { return minIx; } } + + internal char this[int index] + { + get + { + if (index < minIx || index >= maxIx) + throw new BufferException("Index was outside data buffer"); + else if (index < brkIx) + return bldr[index - minIx]; + else + return next[index - brkIx]; + } + } + + internal void Append(char[] block, int count) + { + maxIx += count; + if (appendToNext) + this.next.Append(block, 0, count); + else + { + this.bldr.Append(block, 0, count); + brkIx = maxIx; + appendToNext = true; + } + } + + internal string GetString(int start, int limit) + { + if (limit <= start) + return ""; + if (start >= minIx && limit <= maxIx) + if (limit < brkIx) // String entirely in bldr builder + return bldr.ToString(start - minIx, limit - start); + else if (start >= brkIx) // String entirely in next builder + return next.ToString(start - brkIx, limit - start); + else // Must do a string-concatenation + return + bldr.ToString(start - minIx, brkIx - start) + + next.ToString(0, limit - brkIx); + else + throw new BufferException("String was outside data buffer"); + } + + internal void Mark(int limit) + { + if (limit > brkIx + 16) // Rotate blocks + { + StringBuilder temp = bldr; + bldr = next; + next = temp; + next.Length = 0; + minIx = brkIx; + brkIx = maxIx; + } + } + } + + BufferElement data = new BufferElement(); + + int bPos; // Postion index in the StringBuilder + BlockReader NextBlk; // Delegate that serves char-arrays; + + private string EncodingName + { + get + { + StreamReader rdr = NextBlk.Target as StreamReader; + return (rdr == null ? "raw-bytes" : rdr.CurrentEncoding.BodyName); + } + } + + public BuildBuffer(Stream stream) + { + FileStream fStrm = (stream as FileStream); + if (fStrm != null) FileName = fStrm.Name; + NextBlk = BlockReaderFactory.Raw(stream); + } + +#if (!BYTEMODE) + public BuildBuffer(Stream stream, int fallbackCodePage) + { + FileStream fStrm = (stream as FileStream); + if (fStrm != null) FileName = fStrm.Name; + NextBlk = BlockReaderFactory.Get(stream, fallbackCodePage); + } +#endif + + /// + /// Marks a conservative lower bound for the buffer, + /// allowing space to be reclaimed. If an application + /// needs to call GetString at arbitrary past locations + /// in the input stream, Mark() is not called. + /// + public override void Mark() { data.Mark(bPos - 2); } + + public override int Pos + { + get { return bPos; } + set { bPos = value; } + } + + + /// + /// Read returns the ordinal number of the next char, or + /// EOF (-1) for an end of stream. Note that the next + /// code point may require *two* calls of Read(). + /// + /// + public override int Read() + { + // + // Characters at positions + // [data.offset, data.offset + data.bldr.Length) + // are available in data.bldr. + // + if (bPos < data.MaxIndex) + { + // ch0 cannot be EOF + return (int)data[bPos++]; + } + else // Read from underlying stream + { + // Experimental code, blocks of page size + char[] chrs = new char[4096]; + int count = NextBlk(chrs, 0, 4096); + if (count == 0) + return EndOfFile; + else + { + data.Append(chrs, count); + return (int)data[bPos++]; + } + } + } + + public override string GetString(int begin, int limit) + { + return data.GetString(begin, limit); + } + + public override string ToString() + { + return "StringBuilder buffer, encoding: " + this.EncodingName; + } + } + + // =============== End ScanBuff-derived classes ================== + + internal delegate int BlockReader(char[] block, int index, int number); + + // A delegate factory, serving up a delegate that + // reads a block of characters from the underlying + // encoded stream, via a StreamReader object. + // + internal static class BlockReaderFactory + { + public static BlockReader Raw(Stream stream) + { + return delegate(char[] block, int index, int number) + { + byte[] b = new byte[number]; + int count = stream.Read(b, 0, number); + int i = 0; + int j = index; + for (; i < count; i++, j++) + block[j] = (char)b[i]; + return count; + }; + } + +#if (!BYTEMODE) + public static BlockReader Get(Stream stream, int fallbackCodePage) + { + Encoding encoding; + int preamble = Preamble(stream); + + if (preamble != 0) // There is a valid BOM here! + encoding = Encoding.GetEncoding(preamble); + else if (fallbackCodePage == -1) // Fallback is "raw" bytes + return Raw(stream); + else if (fallbackCodePage != -2) // Anything but "guess" + encoding = Encoding.GetEncoding(fallbackCodePage); + else // This is the "guess" option + { + int guess = new Guesser(stream).GuessCodePage(); + stream.Seek(0, SeekOrigin.Begin); + if (guess == -1) // ==> this is a 7-bit file + encoding = Encoding.ASCII; + else if (guess == 65001) + encoding = Encoding.UTF8; + else // ==> use the machine default + encoding = Encoding.Default; + } + StreamReader reader = new StreamReader(stream, encoding); + return reader.Read; + } + + static int Preamble(Stream stream) + { + int b0 = stream.ReadByte(); + int b1 = stream.ReadByte(); + + if (b0 == 0xfe && b1 == 0xff) + return 1201; // UTF16BE + if (b0 == 0xff && b1 == 0xfe) + return 1200; // UTF16LE + + int b2 = stream.ReadByte(); + if (b0 == 0xef && b1 == 0xbb && b2 == 0xbf) + return 65001; // UTF8 + // + // There is no unicode preamble, so we + // return denoter for the machine default. + // + stream.Seek(0, SeekOrigin.Begin); + return 0; + } +#endif // !BYTEMODE + } +#endif // !NOFILES + #endregion Buffer classes + + // ============================================================== + // ============ class CodePageHandling ============= + // ============================================================== +#if (!NOFILES) + public static class CodePageHandling + { + public static int GetCodePage(string option) + { + string command = option.ToUpperInvariant(); + if (command.StartsWith("CodePage:", StringComparison.OrdinalIgnoreCase)) + command = command.Substring(9); + try + { + if (command.Equals("RAW")) + return -1; + else if (command.Equals("GUESS")) + return -2; + else if (command.Equals("DEFAULT")) + return 0; + else if (char.IsDigit(command[0])) + return int.Parse(command, CultureInfo.InvariantCulture); + else + { + Encoding enc = Encoding.GetEncoding(command); + return enc.CodePage; + } + } + catch (FormatException) + { + Console.Error.WriteLine( + "Invalid format \"{0}\", using machine default", option); + } + catch (ArgumentException) + { + Console.Error.WriteLine( + "Unknown code page \"{0}\", using machine default", option); + } + return 0; + } + } +#region guesser +#if (!BYTEMODE) + // ============================================================== + // ============ Encoding Guesser ============= + // ============================================================== + + /// + /// This class provides a simple finite state automaton that + /// scans the file looking for (1) valid UTF-8 byte patterns, + /// (2) bytes >= 0x80 which are not part of a UTF-8 sequence. + /// The method then guesses whether it is UTF-8 or maybe some + /// local machine default encoding. This works well for the + /// various Latin encodings. + /// + internal class Guesser + { + ScanBuff buffer; + + public int GuessCodePage() { return Scan(); } + + const int maxAccept = 10; + const int initial = 0; + const int eofNum = 0; + const int goStart = -1; + const int INITIAL = 0; + const int EndToken = 0; + + #region user code + /* + * Reads the bytes of a file to determine if it is + * UTF-8 or a single-byte code page file. + */ + public long utfX; + public long uppr; + #endregion user code + + int state; + int currentStart = startState[0]; + int code; + + #region ScannerTables + static int[] startState = new int[] { 11, 0 }; + + #region CharacterMap + static sbyte[] map = new sbyte[256] { +/* '\0' */ 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, +/* '\x10' */ 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, +/* '\x20' */ 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, +/* '0' */ 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, +/* '@' */ 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, +/* 'P' */ 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, +/* '`' */ 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, +/* 'p' */ 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, +/* '\x80' */ 2, 2, 2, 2, 2, 2, 2, 2, 2, 2, 2, 2, 2, 2, 2, 2, +/* '\x90' */ 2, 2, 2, 2, 2, 2, 2, 2, 2, 2, 2, 2, 2, 2, 2, 2, +/* '\xA0' */ 2, 2, 2, 2, 2, 2, 2, 2, 2, 2, 2, 2, 2, 2, 2, 2, +/* '\xB0' */ 2, 2, 2, 2, 2, 2, 2, 2, 2, 2, 2, 2, 2, 2, 2, 2, +/* '\xC0' */ 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, +/* '\xD0' */ 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, +/* '\xE0' */ 3, 3, 3, 3, 3, 3, 3, 3, 3, 3, 3, 3, 3, 3, 3, 3, +/* '\xF0' */ 4, 4, 4, 4, 4, 4, 4, 4, 5, 5, 5, 5, 5, 5, 5, 5 }; + #endregion + + static sbyte[][] nextState = new sbyte[][] { + new sbyte[] {0, 0, 0, 0, 0, 0}, + new sbyte[] {-1, -1, 10, -1, -1, -1}, + new sbyte[] {-1, -1, -1, -1, -1, -1}, + new sbyte[] {-1, -1, 8, -1, -1, -1}, + new sbyte[] {-1, -1, 5, -1, -1, -1}, + new sbyte[] {-1, -1, 6, -1, -1, -1}, + new sbyte[] {-1, -1, 7, -1, -1, -1}, + null, + new sbyte[] {-1, -1, 9, -1, -1, -1}, + null, + null, + new sbyte[] {-1, 1, 2, 3, 4, 2} + }; + + + [SuppressMessage("Microsoft.Performance", "CA1810:InitializeReferenceTypeStaticFieldsInline")] + // Reason for suppression: cannot have self-reference in array initializer. + static Guesser() + { + nextState[7] = nextState[2]; + nextState[9] = nextState[2]; + nextState[10] = nextState[2]; + } + + int NextState() + { + if (code == ScanBuff.EndOfFile) + return eofNum; + else + return nextState[state][map[code]]; + } + #endregion + + public Guesser(System.IO.Stream file) { SetSource(file); } + + public void SetSource(System.IO.Stream source) + { + this.buffer = new BuildBuffer(source); + code = buffer.Read(); + } + + int Scan() + { + for (; ; ) + { + int next; + state = currentStart; + while ((next = NextState()) == goStart) + code = buffer.Read(); + + state = next; + code = buffer.Read(); + + while ((next = NextState()) > eofNum) + { + state = next; + code = buffer.Read(); + } + if (state <= maxAccept) + { + #region ActionSwitch +#pragma warning disable 162 + switch (state) + { + case eofNum: + switch (currentStart) + { + case 11: + if (utfX == 0 && uppr == 0) return -1; /* raw ascii */ + else if (uppr * 10 > utfX) return 0; /* default code page */ + else return 65001; /* UTF-8 encoding */ + break; + } + return EndToken; + case 1: // Recognized '{Upper128}', Shortest string "\xC0" + case 2: // Recognized '{Upper128}', Shortest string "\x80" + case 3: // Recognized '{Upper128}', Shortest string "\xE0" + case 4: // Recognized '{Upper128}', Shortest string "\xF0" + uppr++; + break; + case 5: // Recognized '{Utf8pfx4}{Utf8cont}', Shortest string "\xF0\x80" + uppr += 2; + break; + case 6: // Recognized '{Utf8pfx4}{Utf8cont}{2}', Shortest string "\xF0\x80\x80" + uppr += 3; + break; + case 7: // Recognized '{Utf8pfx4}{Utf8cont}{3}', Shortest string "\xF0\x80\x80\x80" + utfX += 3; + break; + case 8: // Recognized '{Utf8pfx3}{Utf8cont}', Shortest string "\xE0\x80" + uppr += 2; + break; + case 9: // Recognized '{Utf8pfx3}{Utf8cont}{2}', Shortest string "\xE0\x80\x80" + utfX += 2; + break; + case 10: // Recognized '{Utf8pfx2}{Utf8cont}', Shortest string "\xC0\x80" + utfX++; + break; + default: + break; + } +#pragma warning restore 162 + #endregion + } + } + } + } // end class Guesser + +#endif // !BYTEMODE +#endregion +#endif // !NOFILES + +// End of code copied from embedded resource +} diff --git a/src/DataStreamSuite/DataStreamPublisher/Program.cs b/src/DataStreamSuite/DataStreamPublisher/Program.cs index f0eb52994..a86fd2e2a 100644 --- a/src/DataStreamSuite/DataStreamPublisher/Program.cs +++ b/src/DataStreamSuite/DataStreamPublisher/Program.cs @@ -70,6 +70,7 @@ void _publish_streams() { //{"books","books()"}, {"seconds_1_10","second(1,10)"}, {"literal_420","literal(\"x\", \"420.0\")"}, + {"tradebar_30","tradebars(30)"}, {"mongodb_testcollection", "mongodb(\"mongodb://192.168.1.80:27017\",\"local\",\"test_collection\")"} }; // {"bbo_ts","timeSeries(bbo())"}