diff --git a/src/DataStreamSuite/ColumnMapExpressions/ColumnMapExpression.cs b/src/DataStreamSuite/ColumnMapExpressions/ColumnMapExpression.cs new file mode 100644 index 000000000..f7514cb73 --- /dev/null +++ b/src/DataStreamSuite/ColumnMapExpressions/ColumnMapExpression.cs @@ -0,0 +1,26 @@ +namespace Bts.ColumnMapExpressions; + +using Bts.ExpressionCore; + +using System.Collections.Immutable; + +public class ColumnMapExpression { + public ColumnMapExpression(ImmutableDictionary columns, bool isInclusive) { + Columns = columns; + IsInclusive = isInclusive; + } + + public ImmutableDictionary Columns { get; } + public bool IsInclusive { get; } + + public static ColumnMapExpression? Parse(string expression) => + new Parser().Parse(expression); + + public static ColumnMapExpression? Parse(Stream expression) => + new Parser().Parse(expression); + + class Parser : IParser { + public ColumnMapExpression? Parse(Stream expression) => + new ColumnMapExpressionParser().Parse(expression)?.Expression; + } +} diff --git a/src/DataStreamSuite/ColumnMapExpressions/ColumnMapExpressionGrammar.y b/src/DataStreamSuite/ColumnMapExpressions/ColumnMapExpressionGrammar.y new file mode 100644 index 000000000..ae4bc3c2d --- /dev/null +++ b/src/DataStreamSuite/ColumnMapExpressions/ColumnMapExpressionGrammar.y @@ -0,0 +1,40 @@ +%namespace Bts.ColumnMapExpressions +%partial +%visibility internal +%tokentype ColumnMapExpressionToken +%parsertype ColumnMapExpressionParser + +%union { + public string Identifier; + public KeyValuePair MappedColumn; + public System.Collections.Immutable.ImmutableDictionary MappedColumns; + public ColumnMapExpression Expression; +} + +%token IDENTIFIER, MAP, NEGATE + +%type MappedColumns +%type Expression +%type MappedColumn + +%% + +Expression : + NEGATE MappedColumns { $$ = new($2, false); } + | NEGATE { $$ = new(System.Collections.Immutable.ImmutableDictionary.Empty, false); } + | MappedColumns { $$ = new($1, true); } + ; + +MappedColumn : + IDENTIFIER { + $$ = new($1.Identifier, $1.Identifier); + } + | IDENTIFIER MAP IDENTIFIER { + $$ = new($1.Identifier, $3.Identifier); + } + ; + +MappedColumns : + MappedColumn { $$ = System.Collections.Immutable.ImmutableDictionary.Empty.Add($1.Key, $1.Value); } + | MappedColumn MappedColumns { $$ = $2.Add($1.Key, $1.Value); } + ; diff --git a/src/DataStreamSuite/ColumnMapExpressions/ColumnMapExpressionParser.cs b/src/DataStreamSuite/ColumnMapExpressions/ColumnMapExpressionParser.cs new file mode 100644 index 000000000..8c06d04ba --- /dev/null +++ b/src/DataStreamSuite/ColumnMapExpressions/ColumnMapExpressionParser.cs @@ -0,0 +1,18 @@ +namespace Bts.ColumnMapExpressions; + +using System.Text; + +partial class ColumnMapExpressionParser { + public ColumnMapExpressionParser() : base(null) { } + + public ValueType? Parse(string s) { + using var ms = new MemoryStream(Encoding.Default.GetBytes(s)); + return Parse(ms); + } + + public ValueType? Parse(Stream stream) { + Scanner = new ColumnMapExpressionScanner(stream); + Parse(); + return CurrentSemanticValue; + } +} diff --git a/src/DataStreamSuite/ColumnMapExpressions/ColumnMapExpressionTokens.lex b/src/DataStreamSuite/ColumnMapExpressions/ColumnMapExpressionTokens.lex new file mode 100644 index 000000000..bcf4731a0 --- /dev/null +++ b/src/DataStreamSuite/ColumnMapExpressions/ColumnMapExpressionTokens.lex @@ -0,0 +1,30 @@ +%namespace Bts.ColumnMapExpressions +%scannertype ColumnMapExpressionScanner +%visibility internal +%tokentype ColumnMapExpressionToken + +%option stack, minimize, parser, verbose, persistbuffer, noembedbuffers + +/* Literal Constants */ +Letter [a-zA-Z] + +Identifier {Letter}({Letter}*[0-9]*_*)* +Map : +Negate ! + +%% + +{Identifier} { + yylval.Identifier = yytext; + return (int)ColumnMapExpressionToken.IDENTIFIER; +} + +{Map} { + return (int)ColumnMapExpressionToken.MAP; +} + +{Negate} { + return (int)ColumnMapExpressionToken.NEGATE; +} + +%% diff --git a/src/DataStreamSuite/ColumnMapExpressions/ColumnMapExpressions.csproj b/src/DataStreamSuite/ColumnMapExpressions/ColumnMapExpressions.csproj new file mode 100644 index 000000000..4e54ed068 --- /dev/null +++ b/src/DataStreamSuite/ColumnMapExpressions/ColumnMapExpressions.csproj @@ -0,0 +1,25 @@ + + + + net7.0 + enable + enable + + + + + + + + + + + + + + + + + + + diff --git a/src/DataStreamSuite/ColumnMapExpressions/GplexBuffers.cs b/src/DataStreamSuite/ColumnMapExpressions/GplexBuffers.cs new file mode 100644 index 000000000..76dd35443 --- /dev/null +++ b/src/DataStreamSuite/ColumnMapExpressions/GplexBuffers.cs @@ -0,0 +1,720 @@ +// ============================================================== +// +// This code automatically produced from an embedded resource. +// Do not edit this file, or it will become incompatible with +// the specification from which it was generated. +// +// ============================================================== + +using System; +using System.IO; +using System.Text; +using System.Collections.Generic; +using System.Diagnostics.CodeAnalysis; +using System.Runtime.Serialization; +using System.Globalization; + +namespace QUT.GplexBuffers +{ +// Code copied from GPLEX embedded resource + [Serializable] + internal class BufferException : Exception + { + public BufferException() { } + public BufferException(string message) : base(message) { } + public BufferException(string message, Exception innerException) + : base(message, innerException) { } + protected BufferException(SerializationInfo info, StreamingContext context) + : base(info, context) { } + } + + internal abstract class ScanBuff + { + private string fileNm; + + public const int EndOfFile = -1; + public const int UnicodeReplacementChar = 0xFFFD; + + public bool IsFile { get { return (fileNm != null); } } + public string FileName { get { return fileNm; } set { fileNm = value; } } + + public abstract int Pos { get; set; } + public abstract int Read(); + public virtual void Mark() { } + + public abstract string GetString(int begin, int limit); + + public static ScanBuff GetBuffer(string source) + { + return new StringBuffer(source); + } + + public static ScanBuff GetBuffer(IList source) + { + return new LineBuffer(source); + } + +#if (!NOFILES) + public static ScanBuff GetBuffer(Stream source) + { + return new BuildBuffer(source); + } + +#if (!BYTEMODE) + public static ScanBuff GetBuffer(Stream source, int fallbackCodePage) + { + return new BuildBuffer(source, fallbackCodePage); + } +#endif // !BYTEMODE +#endif // !NOFILES + } + + #region Buffer classes + + // ============================================================== + // ===== Definitions for various ScanBuff derived classes ==== + // ============================================================== + // =============== String input ================ + // ============================================================== + + /// + /// This class reads characters from a single string as + /// required, for example, by Visual Studio language services + /// + sealed class StringBuffer : ScanBuff + { + string str; // input buffer + int bPos; // current position in buffer + int sLen; + + public StringBuffer(string source) + { + this.str = source; + this.sLen = source.Length; + this.FileName = null; + } + + public override int Read() + { + if (bPos < sLen) return str[bPos++]; + else if (bPos == sLen) { bPos++; return '\n'; } // one strike, see new line + else { bPos++; return EndOfFile; } // two strikes and you're out! + } + + public override string GetString(int begin, int limit) + { + // "limit" can be greater than sLen with the BABEL + // option set. Read returns a "virtual" EOL if + // an attempt is made to read past the end of the + // string buffer. Without the guard any attempt + // to fetch yytext for a token that includes the + // EOL will throw an index exception. + if (limit > sLen) limit = sLen; + if (limit <= begin) return ""; + else return str.Substring(begin, limit - begin); + } + + public override int Pos + { + get { return bPos; } + set { bPos = value; } + } + + public override string ToString() { return "StringBuffer"; } + } + + // ============================================================== + // The LineBuff class contributed by Nigel Horspool, + // nigelh@cs.uvic.cs + // ============================================================== + + sealed class LineBuffer : ScanBuff + { + IList line; // list of source lines from a file + int numLines; // number of strings in line list + string curLine; // current line in that list + int cLine; // index of current line in the list + int curLen; // length of current line + int curLineStart; // position of line start in whole file + int curLineEnd; // position of line end in whole file + int maxPos; // max position ever visited in whole file + int cPos; // ordinal number of code in source + + // Constructed from a list of strings, one per source line. + // The lines have had trailing '\n' characters removed. + public LineBuffer(IList lineList) + { + line = lineList; + numLines = line.Count; + cPos = curLineStart = 0; + curLine = (numLines > 0 ? line[0] : ""); + maxPos = curLineEnd = curLen = curLine.Length; + cLine = 1; + FileName = null; + } + + public override int Read() + { + if (cPos < curLineEnd) + return curLine[cPos++ - curLineStart]; + if (cPos++ == curLineEnd) + return '\n'; + if (cLine >= numLines) + return EndOfFile; + curLine = line[cLine]; + curLen = curLine.Length; + curLineStart = curLineEnd + 1; + curLineEnd = curLineStart + curLen; + if (curLineEnd > maxPos) + maxPos = curLineEnd; + cLine++; + return curLen > 0 ? curLine[0] : '\n'; + } + + // To speed up searches for the line containing a position + private int cachedPosition; + private int cachedIxdex; + private int cachedLineStart; + + // Given a position pos within the entire source, the results are + // ix -- the index of the containing line + // lstart -- the position of the first character on that line + private void findIndex(int pos, out int ix, out int lstart) + { + if (pos >= cachedPosition) + { + ix = cachedIxdex; lstart = cachedLineStart; + } + else + { + ix = lstart = 0; + } + while (ix < numLines) + { + int len = line[ix].Length + 1; + if (pos < lstart + len) break; + lstart += len; + ix++; + } + cachedPosition = pos; + cachedIxdex = ix; + cachedLineStart = lstart; + } + + public override string GetString(int begin, int limit) + { + if (begin >= maxPos || limit <= begin) return ""; + int endIx, begIx, endLineStart, begLineStart; + findIndex(begin, out begIx, out begLineStart); + int begCol = begin - begLineStart; + findIndex(limit, out endIx, out endLineStart); + int endCol = limit - endLineStart; + string s = line[begIx]; + if (begIx == endIx) + { + // the usual case, substring all on one line + return (endCol <= s.Length) ? + s.Substring(begCol, endCol - begCol) + : s.Substring(begCol) + "\n"; + } + // the string spans multiple lines, yuk! + StringBuilder sb = new StringBuilder(); + if (begCol < s.Length) + sb.Append(s.Substring(begCol)); + for (; ; ) + { + sb.Append("\n"); + s = line[++begIx]; + if (begIx >= endIx) break; + sb.Append(s); + } + if (endCol <= s.Length) + { + sb.Append(s.Substring(0, endCol)); + } + else + { + sb.Append(s); + sb.Append("\n"); + } + return sb.ToString(); + } + + public override int Pos + { + get { return cPos; } + set + { + cPos = value; + findIndex(cPos, out cLine, out curLineStart); + // cLine should be the *next* line after curLine. + curLine = (cLine < numLines ? line[cLine++] : ""); + curLineEnd = curLineStart + curLine.Length; + } + } + + public override string ToString() { return "LineBuffer"; } + } + +#if (!NOFILES) + // ============================================================== + // ===== class BuildBuff : for unicode text files ======== + // ============================================================== + + class BuildBuffer : ScanBuff + { + // Double buffer for char stream. + class BufferElement + { + StringBuilder bldr = new StringBuilder(); + StringBuilder next = new StringBuilder(); + int minIx; + int maxIx; + int brkIx; + bool appendToNext; + + internal BufferElement() { } + + internal int MaxIndex { get { return maxIx; } } + // internal int MinIndex { get { return minIx; } } + + internal char this[int index] + { + get + { + if (index < minIx || index >= maxIx) + throw new BufferException("Index was outside data buffer"); + else if (index < brkIx) + return bldr[index - minIx]; + else + return next[index - brkIx]; + } + } + + internal void Append(char[] block, int count) + { + maxIx += count; + if (appendToNext) + this.next.Append(block, 0, count); + else + { + this.bldr.Append(block, 0, count); + brkIx = maxIx; + appendToNext = true; + } + } + + internal string GetString(int start, int limit) + { + if (limit <= start) + return ""; + if (start >= minIx && limit <= maxIx) + if (limit < brkIx) // String entirely in bldr builder + return bldr.ToString(start - minIx, limit - start); + else if (start >= brkIx) // String entirely in next builder + return next.ToString(start - brkIx, limit - start); + else // Must do a string-concatenation + return + bldr.ToString(start - minIx, brkIx - start) + + next.ToString(0, limit - brkIx); + else + throw new BufferException("String was outside data buffer"); + } + + internal void Mark(int limit) + { + if (limit > brkIx + 16) // Rotate blocks + { + StringBuilder temp = bldr; + bldr = next; + next = temp; + next.Length = 0; + minIx = brkIx; + brkIx = maxIx; + } + } + } + + BufferElement data = new BufferElement(); + + int bPos; // Postion index in the StringBuilder + BlockReader NextBlk; // Delegate that serves char-arrays; + + private string EncodingName + { + get + { + StreamReader rdr = NextBlk.Target as StreamReader; + return (rdr == null ? "raw-bytes" : rdr.CurrentEncoding.BodyName); + } + } + + public BuildBuffer(Stream stream) + { + FileStream fStrm = (stream as FileStream); + if (fStrm != null) FileName = fStrm.Name; + NextBlk = BlockReaderFactory.Raw(stream); + } + +#if (!BYTEMODE) + public BuildBuffer(Stream stream, int fallbackCodePage) + { + FileStream fStrm = (stream as FileStream); + if (fStrm != null) FileName = fStrm.Name; + NextBlk = BlockReaderFactory.Get(stream, fallbackCodePage); + } +#endif + + /// + /// Marks a conservative lower bound for the buffer, + /// allowing space to be reclaimed. If an application + /// needs to call GetString at arbitrary past locations + /// in the input stream, Mark() is not called. + /// + public override void Mark() { data.Mark(bPos - 2); } + + public override int Pos + { + get { return bPos; } + set { bPos = value; } + } + + + /// + /// Read returns the ordinal number of the next char, or + /// EOF (-1) for an end of stream. Note that the next + /// code point may require *two* calls of Read(). + /// + /// + public override int Read() + { + // + // Characters at positions + // [data.offset, data.offset + data.bldr.Length) + // are available in data.bldr. + // + if (bPos < data.MaxIndex) + { + // ch0 cannot be EOF + return (int)data[bPos++]; + } + else // Read from underlying stream + { + // Experimental code, blocks of page size + char[] chrs = new char[4096]; + int count = NextBlk(chrs, 0, 4096); + if (count == 0) + return EndOfFile; + else + { + data.Append(chrs, count); + return (int)data[bPos++]; + } + } + } + + public override string GetString(int begin, int limit) + { + return data.GetString(begin, limit); + } + + public override string ToString() + { + return "StringBuilder buffer, encoding: " + this.EncodingName; + } + } + + // =============== End ScanBuff-derived classes ================== + + internal delegate int BlockReader(char[] block, int index, int number); + + // A delegate factory, serving up a delegate that + // reads a block of characters from the underlying + // encoded stream, via a StreamReader object. + // + internal static class BlockReaderFactory + { + public static BlockReader Raw(Stream stream) + { + return delegate(char[] block, int index, int number) + { + byte[] b = new byte[number]; + int count = stream.Read(b, 0, number); + int i = 0; + int j = index; + for (; i < count; i++, j++) + block[j] = (char)b[i]; + return count; + }; + } + +#if (!BYTEMODE) + public static BlockReader Get(Stream stream, int fallbackCodePage) + { + Encoding encoding; + int preamble = Preamble(stream); + + if (preamble != 0) // There is a valid BOM here! + encoding = Encoding.GetEncoding(preamble); + else if (fallbackCodePage == -1) // Fallback is "raw" bytes + return Raw(stream); + else if (fallbackCodePage != -2) // Anything but "guess" + encoding = Encoding.GetEncoding(fallbackCodePage); + else // This is the "guess" option + { + int guess = new Guesser(stream).GuessCodePage(); + stream.Seek(0, SeekOrigin.Begin); + if (guess == -1) // ==> this is a 7-bit file + encoding = Encoding.ASCII; + else if (guess == 65001) + encoding = Encoding.UTF8; + else // ==> use the machine default + encoding = Encoding.Default; + } + StreamReader reader = new StreamReader(stream, encoding); + return reader.Read; + } + + static int Preamble(Stream stream) + { + int b0 = stream.ReadByte(); + int b1 = stream.ReadByte(); + + if (b0 == 0xfe && b1 == 0xff) + return 1201; // UTF16BE + if (b0 == 0xff && b1 == 0xfe) + return 1200; // UTF16LE + + int b2 = stream.ReadByte(); + if (b0 == 0xef && b1 == 0xbb && b2 == 0xbf) + return 65001; // UTF8 + // + // There is no unicode preamble, so we + // return denoter for the machine default. + // + stream.Seek(0, SeekOrigin.Begin); + return 0; + } +#endif // !BYTEMODE + } +#endif // !NOFILES + #endregion Buffer classes + + // ============================================================== + // ============ class CodePageHandling ============= + // ============================================================== +#if (!NOFILES) + public static class CodePageHandling + { + public static int GetCodePage(string option) + { + string command = option.ToUpperInvariant(); + if (command.StartsWith("CodePage:", StringComparison.OrdinalIgnoreCase)) + command = command.Substring(9); + try + { + if (command.Equals("RAW")) + return -1; + else if (command.Equals("GUESS")) + return -2; + else if (command.Equals("DEFAULT")) + return 0; + else if (char.IsDigit(command[0])) + return int.Parse(command, CultureInfo.InvariantCulture); + else + { + Encoding enc = Encoding.GetEncoding(command); + return enc.CodePage; + } + } + catch (FormatException) + { + Console.Error.WriteLine( + "Invalid format \"{0}\", using machine default", option); + } + catch (ArgumentException) + { + Console.Error.WriteLine( + "Unknown code page \"{0}\", using machine default", option); + } + return 0; + } + } +#region guesser +#if (!BYTEMODE) + // ============================================================== + // ============ Encoding Guesser ============= + // ============================================================== + + /// + /// This class provides a simple finite state automaton that + /// scans the file looking for (1) valid UTF-8 byte patterns, + /// (2) bytes >= 0x80 which are not part of a UTF-8 sequence. + /// The method then guesses whether it is UTF-8 or maybe some + /// local machine default encoding. This works well for the + /// various Latin encodings. + /// + internal class Guesser + { + ScanBuff buffer; + + public int GuessCodePage() { return Scan(); } + + const int maxAccept = 10; + const int initial = 0; + const int eofNum = 0; + const int goStart = -1; + const int INITIAL = 0; + const int EndToken = 0; + + #region user code + /* + * Reads the bytes of a file to determine if it is + * UTF-8 or a single-byte code page file. + */ + public long utfX; + public long uppr; + #endregion user code + + int state; + int currentStart = startState[0]; + int code; + + #region ScannerTables + static int[] startState = new int[] { 11, 0 }; + + #region CharacterMap + static sbyte[] map = new sbyte[256] { +/* '\0' */ 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, +/* '\x10' */ 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, +/* '\x20' */ 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, +/* '0' */ 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, +/* '@' */ 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, +/* 'P' */ 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, +/* '`' */ 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, +/* 'p' */ 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, +/* '\x80' */ 2, 2, 2, 2, 2, 2, 2, 2, 2, 2, 2, 2, 2, 2, 2, 2, +/* '\x90' */ 2, 2, 2, 2, 2, 2, 2, 2, 2, 2, 2, 2, 2, 2, 2, 2, +/* '\xA0' */ 2, 2, 2, 2, 2, 2, 2, 2, 2, 2, 2, 2, 2, 2, 2, 2, +/* '\xB0' */ 2, 2, 2, 2, 2, 2, 2, 2, 2, 2, 2, 2, 2, 2, 2, 2, +/* '\xC0' */ 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, +/* '\xD0' */ 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, +/* '\xE0' */ 3, 3, 3, 3, 3, 3, 3, 3, 3, 3, 3, 3, 3, 3, 3, 3, +/* '\xF0' */ 4, 4, 4, 4, 4, 4, 4, 4, 5, 5, 5, 5, 5, 5, 5, 5 }; + #endregion + + static sbyte[][] nextState = new sbyte[][] { + new sbyte[] {0, 0, 0, 0, 0, 0}, + new sbyte[] {-1, -1, 10, -1, -1, -1}, + new sbyte[] {-1, -1, -1, -1, -1, -1}, + new sbyte[] {-1, -1, 8, -1, -1, -1}, + new sbyte[] {-1, -1, 5, -1, -1, -1}, + new sbyte[] {-1, -1, 6, -1, -1, -1}, + new sbyte[] {-1, -1, 7, -1, -1, -1}, + null, + new sbyte[] {-1, -1, 9, -1, -1, -1}, + null, + null, + new sbyte[] {-1, 1, 2, 3, 4, 2} + }; + + + [SuppressMessage("Microsoft.Performance", "CA1810:InitializeReferenceTypeStaticFieldsInline")] + // Reason for suppression: cannot have self-reference in array initializer. + static Guesser() + { + nextState[7] = nextState[2]; + nextState[9] = nextState[2]; + nextState[10] = nextState[2]; + } + + int NextState() + { + if (code == ScanBuff.EndOfFile) + return eofNum; + else + return nextState[state][map[code]]; + } + #endregion + + public Guesser(System.IO.Stream file) { SetSource(file); } + + public void SetSource(System.IO.Stream source) + { + this.buffer = new BuildBuffer(source); + code = buffer.Read(); + } + + int Scan() + { + for (; ; ) + { + int next; + state = currentStart; + while ((next = NextState()) == goStart) + code = buffer.Read(); + + state = next; + code = buffer.Read(); + + while ((next = NextState()) > eofNum) + { + state = next; + code = buffer.Read(); + } + if (state <= maxAccept) + { + #region ActionSwitch +#pragma warning disable 162 + switch (state) + { + case eofNum: + switch (currentStart) + { + case 11: + if (utfX == 0 && uppr == 0) return -1; /* raw ascii */ + else if (uppr * 10 > utfX) return 0; /* default code page */ + else return 65001; /* UTF-8 encoding */ + break; + } + return EndToken; + case 1: // Recognized '{Upper128}', Shortest string "\xC0" + case 2: // Recognized '{Upper128}', Shortest string "\x80" + case 3: // Recognized '{Upper128}', Shortest string "\xE0" + case 4: // Recognized '{Upper128}', Shortest string "\xF0" + uppr++; + break; + case 5: // Recognized '{Utf8pfx4}{Utf8cont}', Shortest string "\xF0\x80" + uppr += 2; + break; + case 6: // Recognized '{Utf8pfx4}{Utf8cont}{2}', Shortest string "\xF0\x80\x80" + uppr += 3; + break; + case 7: // Recognized '{Utf8pfx4}{Utf8cont}{3}', Shortest string "\xF0\x80\x80\x80" + utfX += 3; + break; + case 8: // Recognized '{Utf8pfx3}{Utf8cont}', Shortest string "\xE0\x80" + uppr += 2; + break; + case 9: // Recognized '{Utf8pfx3}{Utf8cont}{2}', Shortest string "\xE0\x80\x80" + utfX += 2; + break; + case 10: // Recognized '{Utf8pfx2}{Utf8cont}', Shortest string "\xC0\x80" + utfX++; + break; + default: + break; + } +#pragma warning restore 162 + #endregion + } + } + } + } // end class Guesser + +#endif // !BYTEMODE +#endregion +#endif // !NOFILES + +// End of code copied from embedded resource +} diff --git a/src/DataStreamSuite/ColumnMapExpressions/ShiftReduceParserCode.cs b/src/DataStreamSuite/ColumnMapExpressions/ShiftReduceParserCode.cs new file mode 100644 index 000000000..a255acb0d --- /dev/null +++ b/src/DataStreamSuite/ColumnMapExpressions/ShiftReduceParserCode.cs @@ -0,0 +1,884 @@ +// ============================================================== +// +// This code automatically produced from an embedded resource. +// Do not edit this file, or it will become incompatible with +// the specification from which it was generated. +// +// ============================================================== + +// Gardens Point Parser Generator +// Copyright (c) Wayne Kelly, QUT 2005-2010 +// (see accompanying GPPGcopyright.rtf) + +using System; +using System.Text; +using System.Globalization; +using System.Collections.Generic; +using System.Runtime.Serialization; +using System.Diagnostics.CodeAnalysis; + +namespace QUT.Gppg { + /// + /// Abstract class for GPPG shift-reduce parsers. + /// Parsers generated by GPPG derive from this base + /// class, overriding the abstract Initialize() and + /// DoAction() methods. + /// + /// Semantic value type + /// Location type +#if EXPORT_GPPG + public abstract class ShiftReduceParser +#else + internal abstract class ShiftReduceParser +#endif + where TSpan : IMerge, new() { + private AbstractScanner scanner; + /// + /// The abstract scanner for this parser. + /// + protected AbstractScanner Scanner { + get { return scanner; } + set { scanner = value; } + } + + /// + /// Constructor for base class + /// + /// Scanner instance for this parser + protected ShiftReduceParser( AbstractScanner scanner ) { + this.scanner = scanner; + } + + // ============================================================== + // TECHNICAL EXPLANATION. + // Why the next two fields are not exposed via properties. + // ============================================================== + // These fields are of the generic parameter types, and are + // frequently instantiated as struct types in derived classes. + // Semantic actions are defined in the derived classes and refer + // to instance fields of these structs. In such cases the code + // "get_CurrentSemanticValue().myField = blah;" will fail since + // the getter pushes the value of the field, not the reference. + // So, in the presence of properties, gppg would need to encode + // such field accesses as ... + // "tmp = get_CurrentSemanticValue(); // Fetch value + // tmp.myField = blah; // update + // set_CurrentSemanticValue(tmp); " // Write update back. + // There is no issue if TValue is restricted to be a ref type. + // The same explanation applies to scanner.yylval. + // ============================================================== + /// + /// The current value of the "$$" symbolic variable in the parser + /// + [SuppressMessage( "Microsoft.Design", "CA1051:DoNotDeclareVisibleInstanceFields" )] + protected TValue CurrentSemanticValue; + + /// + /// The current value of the "@$" symbolic variable in the parser + /// + [SuppressMessage( "Microsoft.Design", "CA1051:DoNotDeclareVisibleInstanceFields" )] + protected TSpan CurrentLocationSpan; + protected int NextToken; + + private TSpan LastSpan; + private State FsaState; + private bool recovering; + private int tokensSinceLastError; + + private PushdownPrefixState StateStack = new PushdownPrefixState(); + private PushdownPrefixState valueStack = new PushdownPrefixState(); + private PushdownPrefixState locationStack = new PushdownPrefixState(); + + /// + /// The stack of semantic value (YYSTYPE) values. + /// + protected PushdownPrefixState ValueStack { get { return valueStack; } } + + /// + /// The stack of location value (YYLTYPE) varlues. + /// + protected PushdownPrefixState LocationStack { get { return locationStack; } } + + private int errorToken; + private int endOfFileToken; + private string[] nonTerminals; + private State[] states; + private Rule[] rules; + + /// + /// Initialization method to allow derived classes + /// to insert the rule list into this base class. + /// + /// The array of Rule objects + protected void InitRules( Rule[] rules ) { this.rules = rules; } + + /// + /// Initialization method to allow derived classes to + /// insert the states table into this base class. + /// + /// The pre-initialized states table + protected void InitStates( State[] states ) { this.states = states; } + + /// + /// OBSOLETE FOR VERSION 1.4.0 + /// + /// + protected void InitStateTable( int size ) { states = new State[size]; } + + /// + /// Initialization method to allow derived classes + /// to insert the special value for the error and EOF tokens. + /// + /// The error state ordinal + /// The EOF stat ordinal + protected void InitSpecialTokens( int err, int end ) { + errorToken = err; + endOfFileToken = end; + } + + /// + /// Initialization method to allow derived classes to + /// insert the non-terminal symbol names into this base class. + /// + /// Non-terminal symbol names + protected void InitNonTerminals( string[] names ) { nonTerminals = names; } + + #region YYAbort, YYAccept etcetera. + [Serializable] + [SuppressMessage( "Microsoft.Design", "CA1064:ExceptionsShouldBePublic" )] + // Reason for FxCop message suppression - + // This exception cannot escape from the local context + private class AcceptException : Exception { + internal AcceptException() { } + protected AcceptException( SerializationInfo i, StreamingContext c ) : base( i, c ) { } + } + [Serializable] + [SuppressMessage( "Microsoft.Design", "CA1064:ExceptionsShouldBePublic" )] + // Reason for FxCop message suppression - + // This exception cannot escape from the local context + private class AbortException : Exception { + internal AbortException() { } + protected AbortException( SerializationInfo i, StreamingContext c ) : base( i, c ) { } + } + [Serializable] + [SuppressMessage( "Microsoft.Design", "CA1064:ExceptionsShouldBePublic" )] + // Reason for FxCop message suppression - + // This exception cannot escape from the local context + private class ErrorException : Exception { + internal ErrorException() { } + protected ErrorException( SerializationInfo i, StreamingContext c ) : base( i, c ) { } + } + + // The following methods are only called from within + // a semantic action. The thrown exceptions can never + // propagate outside the ShiftReduceParser class in + // which they are nested. + + /// + /// Force parser to terminate, returning "true" + /// + protected static void YYAccept() { throw new AcceptException(); } + + /// + /// Force parser to terminate, returning "false" + /// + protected static void YYAbort() { throw new AbortException(); } + + /// + /// Force parser to terminate, returning + /// "false" if error recovery fails. + /// + protected static void YYError() { throw new ErrorException(); } + + /// + /// Check if parser in error recovery state. + /// + protected bool YYRecovering { get { return recovering; } } + #endregion + + /// + /// Abstract base method. ShiftReduceParser calls this + /// to initialize the base class data structures. Concrete + /// parser classes must override this method. + /// + protected abstract void Initialize(); + + /// + /// Main entry point of the Shift-Reduce Parser. + /// + /// True if parse succeeds, else false for + /// unrecoverable errors + public bool Parse() { + Initialize(); // allow derived classes to instantiate rules, states and nonTerminals + + NextToken = 0; + FsaState = states[0]; + + StateStack.Push( FsaState ); + valueStack.Push( CurrentSemanticValue ); + LocationStack.Push( CurrentLocationSpan ); + + while (true) { +#if TRACE_ACTIONS + Console.Error.WriteLine("Entering state {0} ", FsaState.number); + DisplayStack(); +#endif + int action = FsaState.defaultAction; + + if (FsaState.ParserTable != null) { + if (NextToken == 0) { + // We save the last token span, so that the location span + // of production right hand sides that begin or end with a + // nullable production will be correct. + LastSpan = scanner.yylloc; + NextToken = scanner.yylex(); +#if TRACE_ACTIONS + Console.Error.WriteLine( "Reading: Next token is {0}", TerminalToString( NextToken ) ); +#endif + } +#if TRACE_ACTIONS + else + Console.Error.WriteLine( "Next token is still {0}", TerminalToString( NextToken ) ); +#endif + if (FsaState.ParserTable.ContainsKey( NextToken )) + action = FsaState.ParserTable[NextToken]; + } + + if (action > 0) // shift + { + Shift( action ); + } + else if (action < 0) // reduce + { + try { + Reduce( -action ); + if (action == -1) // accept + return true; + } + catch (AbortException) { + return false; + } + catch (AcceptException) { + return true; + } + catch (ErrorException) { + if (!ErrorRecovery()) + return false; + else + throw; // Rethrow x, preserving information. + } + } + else if (action == 0 && !ErrorRecovery()) // error + return false; + } + } + + private void Shift( int stateIndex ) { +#if TRACE_ACTIONS + Console.Error.Write("Shifting token {0}, ", TerminalToString(NextToken)); +#endif + FsaState = states[stateIndex]; + + valueStack.Push( scanner.yylval ); + StateStack.Push( FsaState ); + LocationStack.Push( scanner.yylloc ); + + if (recovering) { + if (NextToken != errorToken) + tokensSinceLastError++; + + if (tokensSinceLastError > 5) + recovering = false; + } + + if (NextToken != endOfFileToken) + NextToken = 0; + } + + private void Reduce( int ruleNumber ) { +#if TRACE_ACTIONS + DisplayRule(ruleNumber); +#endif + Rule rule = rules[ruleNumber]; + int rhLen = rule.RightHandSide.Length; + // + // Default actions for unit productions. + // + if (rhLen == 1) { + CurrentSemanticValue = valueStack.TopElement(); // Default action: $$ = $1; + CurrentLocationSpan = LocationStack.TopElement(); // Default action "@$ = @1; + } + else if (rhLen == 0) { + // Create a new blank value. + // Explicit semantic action may mutate this value + CurrentSemanticValue = default( TValue ); + // The location span for an empty production will start with the + // beginning of the next lexeme, and end with the finish of the + // previous lexeme. This gives the correct behaviour when this + // nonsense value is used in later Merge operations. + CurrentLocationSpan = (scanner.yylloc != null && LastSpan != null ? + scanner.yylloc.Merge( LastSpan ) : + default( TSpan )); + } + else { + // Default action: $$ = $1; + CurrentSemanticValue = valueStack[LocationStack.Depth - rhLen]; + // Default action "@$ = @1.Merge(@N)" for location info. + TSpan at1 = LocationStack[LocationStack.Depth - rhLen]; + TSpan atN = LocationStack[LocationStack.Depth - 1]; + CurrentLocationSpan = + ((at1 != null && atN != null) ? at1.Merge( atN ) : default( TSpan )); + } + + DoAction( ruleNumber ); + + for (int i = 0; i < rule.RightHandSide.Length; i++) { + StateStack.Pop(); + valueStack.Pop(); + LocationStack.Pop(); + } + FsaState = StateStack.TopElement(); + + if (FsaState.Goto.ContainsKey( rule.LeftHandSide )) + FsaState = states[FsaState.Goto[rule.LeftHandSide]]; + + StateStack.Push( FsaState ); + valueStack.Push( CurrentSemanticValue ); + LocationStack.Push( CurrentLocationSpan ); + } + + /// + /// Execute the selected action from array. + /// Must be overriden in derived classes. + /// + /// Index of the action to perform + protected abstract void DoAction( int actionNumber ); + + private bool ErrorRecovery() { + bool discard; + + if (!recovering) // if not recovering from previous error + ReportError(); + + if (!FindErrorRecoveryState()) + return false; + // + // The interim fix for the "looping in error recovery" + // artifact involved moving the setting of the recovering + // bool until after invalid tokens have been discarded. + // + ShiftErrorToken(); + discard = DiscardInvalidTokens(); + recovering = true; + tokensSinceLastError = 0; + return discard; + } + + private void ReportError() { + StringBuilder errorMsg = new StringBuilder(); + errorMsg.AppendFormat( "Syntax error, unexpected {0}", TerminalToString( NextToken ) ); + + if (FsaState.ParserTable.Count < 7) { + bool first = true; + foreach (int terminal in FsaState.ParserTable.Keys) { + if (first) + errorMsg.Append( ", expecting " ); + else + errorMsg.Append( ", or " ); + + errorMsg.Append( TerminalToString( terminal ) ); + first = false; + } + } + scanner.yyerror( errorMsg.ToString() ); + } + + private void ShiftErrorToken() { + int old_next = NextToken; + NextToken = errorToken; + + Shift( FsaState.ParserTable[NextToken] ); + +#if TRACE_ACTIONS + Console.Error.WriteLine("Entering state {0} ", FsaState.number); +#endif + NextToken = old_next; + } + + private bool FindErrorRecoveryState() { + while (true) // pop states until one found that accepts error token + { + if (FsaState.ParserTable != null && + FsaState.ParserTable.ContainsKey( errorToken ) && + FsaState.ParserTable[errorToken] > 0) // shift + return true; + +#if TRACE_ACTIONS + Console.Error.WriteLine("Error: popping state {0}", StateStack.TopElement().number); +#endif + StateStack.Pop(); + valueStack.Pop(); + LocationStack.Pop(); + +#if TRACE_ACTIONS + DisplayStack(); +#endif + if (StateStack.IsEmpty()) { +#if TRACE_ACTIONS + Console.Error.WriteLine("Aborting: didn't find a state that accepts error token"); +#endif + return false; + } + else + FsaState = StateStack.TopElement(); + } + } + + private bool DiscardInvalidTokens() { + + int action = FsaState.defaultAction; + + if (FsaState.ParserTable != null) { + // Discard tokens until find one that works ... + while (true) { + if (NextToken == 0) { +#if TRACE_ACTIONS + Console.Error.Write("Reading a token: "); +#endif + NextToken = scanner.yylex(); + } +#if TRACE_ACTIONS + Console.Error.WriteLine("Next token is {0}", TerminalToString(NextToken)); +#endif + if (NextToken == endOfFileToken) + return false; + + if (FsaState.ParserTable.ContainsKey( NextToken )) + action = FsaState.ParserTable[NextToken]; + + if (action != 0) + return true; + else { +#if TRACE_ACTIONS + Console.Error.WriteLine("Error: Discarding {0}", TerminalToString(NextToken)); +#endif + NextToken = 0; + } + } + } + else if (recovering && tokensSinceLastError == 0) { + // + // Boolean recovering is not set until after the first + // error token has been shifted. Thus if we get back + // here with recovering set and no tokens read we are + // looping on the same error recovery action. This + // happens if current_state.ParserTable is null because + // the state has an LR(0) reduction, but not all + // lookahead tokens are valid. This only occurs for + // error productions that *end* on "error". + // + // This action discards tokens one at a time until + // the looping stops. Another attack would be to always + // use the LALR(1) table if a production ends on "error" + // +#if TRACE_ACTIONS + Console.Error.WriteLine("Error: panic discard of {0}", TerminalToString(NextToken)); +#endif + if (NextToken == endOfFileToken) + return false; + NextToken = 0; + return true; + } + else + return true; + } + + /// + /// Traditional YACC method. Discards the next input token. + /// + [SuppressMessage( "Microsoft.Naming", "CA1709:IdentifiersShouldBeCasedCorrectly", MessageId = "yyclearin" )] + [SuppressMessage( "Microsoft.Naming", "CA1704:IdentifiersShouldBeSpelledCorrectly", MessageId = "yyclearin" )] + // Reason for FxCop message suppression - + // This is a traditional name for YACC-like functionality + protected void yyclearin() { NextToken = 0; } + + /// + /// Tradional YACC method. Clear the "recovering" flag. + /// + [SuppressMessage( "Microsoft.Naming", "CA1709:IdentifiersShouldBeCasedCorrectly", MessageId = "yyerrok" )] + [SuppressMessage( "Microsoft.Naming", "CA1704:IdentifiersShouldBeSpelledCorrectly", MessageId = "yyerrok" )] + // Reason for FxCop message suppression - + // This is a traditional name for YACC-like functionality + protected void yyerrok() { recovering = false; } + + /// + /// OBSOLETE FOR VERSION 1.4.0 + /// Method used by derived types to insert new + /// state instances in the "states" array. + /// + /// index of the state + /// data for the state + protected void AddState( int stateNumber, State state ) { + states[stateNumber] = state; + state.number = stateNumber; + } + + private void DisplayStack() { + Console.Error.Write( "State stack is now:" ); + for (int i = 0; i < StateStack.Depth; i++) + Console.Error.Write( " {0}", StateStack[i].number ); + Console.Error.WriteLine(); + } + + private void DisplayRule( int ruleNumber ) { + Console.Error.Write( "Reducing stack by rule {0}, ", ruleNumber ); + DisplayProduction( rules[ruleNumber] ); + } + + private void DisplayProduction( Rule rule ) { + if (rule.RightHandSide.Length == 0) + Console.Error.Write( "/* empty */ " ); + else + foreach (int symbol in rule.RightHandSide) + Console.Error.Write( "{0} ", SymbolToString( symbol ) ); + + Console.Error.WriteLine( "-> {0}", SymbolToString( rule.LeftHandSide ) ); + } + + /// + /// Abstract state class naming terminal symbols. + /// This is overridden by derived classes with the + /// name (or alias) to be used in error messages. + /// + /// The terminal ordinal + /// + protected abstract string TerminalToString( int terminal ); + + private string SymbolToString( int symbol ) { + if (symbol < 0) + return nonTerminals[-symbol - 1]; + else + return TerminalToString( symbol ); + } + + /// + /// Return text representation of argument character + /// + /// The character to convert + /// String representation of the character + protected static string CharToString( char input ) { + switch (input) { + case '\a': return @"'\a'"; + case '\b': return @"'\b'"; + case '\f': return @"'\f'"; + case '\n': return @"'\n'"; + case '\r': return @"'\r'"; + case '\t': return @"'\t'"; + case '\v': return @"'\v'"; + case '\0': return @"'\0'"; + default: return string.Format( CultureInfo.InvariantCulture, "'{0}'", input ); + } + } + } + + /// + /// Classes implementing this interface must supply a + /// method that merges two location objects to return + /// a new object of the same type. + /// GPPG-generated parsers have the default location + /// action equivalent to "@$ = @1.Merge(@N);" where N + /// is the right-hand-side length of the production. + /// + /// The Location type +#if EXPORT_GPPG + public interface IMerge +#else + internal interface IMerge +#endif + { + /// + /// Interface method that creates a location object from + /// the current and last object. Typically used to create + /// a location object extending from the start of the @1 + /// object to the end of the @N object. + /// + /// The lexically last object to merge + /// The merged location object + TSpan Merge( TSpan last ); + } + + /// + /// This is the default class that carries location + /// information from the scanner to the parser. + /// If you don't declare "%YYLTYPE Foo" the parser + /// will expect to deal with this type. + /// +#if EXPORT_GPPG + public class LexLocation : IMerge +#else + [SuppressMessage( "Microsoft.Performance", "CA1812:AvoidUninstantiatedInternalClasses" )] + internal class LexLocation : IMerge +#endif + { + private int startLine; // start line + private int startColumn; // start column + private int endLine; // end line + private int endColumn; // end column + + /// + /// The line at which the text span starts. + /// + public int StartLine { get { return startLine; } } + + /// + /// The column at which the text span starts. + /// + public int StartColumn { get { return startColumn; } } + + /// + /// The line on which the text span ends. + /// + public int EndLine { get { return endLine; } } + + /// + /// The column of the first character + /// beyond the end of the text span. + /// + public int EndColumn { get { return endColumn; } } + + /// + /// Default no-arg constructor. + /// + public LexLocation() { } + + /// + /// Constructor for text-span with given start and end. + /// + /// start line + /// start column + /// end line + /// end column + public LexLocation( int sl, int sc, int el, int ec ) { startLine = sl; startColumn = sc; endLine = el; endColumn = ec; } + + /// + /// Create a text location which spans from the + /// start of "this" to the end of the argument "last" + /// + /// The last location in the result span + /// The merged span + public LexLocation Merge( LexLocation last ) { return new LexLocation( this.startLine, this.startColumn, last.endLine, last.endColumn ); } + } + + /// + /// Abstract scanner class that GPPG expects its scanners to + /// extend. + /// + /// Semantic value type YYSTYPE + /// Source location type YYLTYPE +#if EXPORT_GPPG + public abstract class AbstractScanner +#else + internal abstract class AbstractScanner +#endif + where TSpan : IMerge { + /// + /// Lexical value optionally set by the scanner. The value + /// is of the %YYSTYPE type declared in the parser spec. + /// + [SuppressMessage( "Microsoft.Design", "CA1051:DoNotDeclareVisibleInstanceFields" )] + [SuppressMessage( "Microsoft.Naming", "CA1709:IdentifiersShouldBeCasedCorrectly", MessageId = "yylval" )] + [SuppressMessage( "Microsoft.Naming", "CA1704:IdentifiersShouldBeSpelledCorrectly", MessageId = "yylval" )] + // Reason for FxCop message suppression - + // This is a traditional name for YACC-like functionality + // A field must be declared for this value of parametric type, + // since it may be instantiated by a value struct. If it were + // implemented as a property, machine generated code in derived + // types would not be able to select on the returned value. +#pragma warning disable 649 + public TValue yylval; // Lexical value: set by scanner +#pragma warning restore 649 + + /// + /// Current scanner location property. The value is of the + /// type declared by %YYLTYPE in the parser specification. + /// + [SuppressMessage( "Microsoft.Naming", "CA1709:IdentifiersShouldBeCasedCorrectly", MessageId = "yylloc" )] + [SuppressMessage( "Microsoft.Naming", "CA1704:IdentifiersShouldBeSpelledCorrectly", MessageId = "yylloc" )] + // Reason for FxCop message suppression - + // This is a traditional name for YACC-like functionality + public virtual TSpan yylloc { + get { return default( TSpan ); } // Empty implementation allowing + set { /* skip */ } // yylloc to be ignored entirely. + } + + /// + /// Main call point for LEX-like scanners. Returns an int + /// corresponding to the token recognized by the scanner. + /// + /// An int corresponding to the token + [SuppressMessage( "Microsoft.Naming", "CA1709:IdentifiersShouldBeCasedCorrectly", MessageId = "yylex" )] + [SuppressMessage( "Microsoft.Naming", "CA1704:IdentifiersShouldBeSpelledCorrectly", MessageId = "yylex" )] + // Reason for FxCop message suppression - + // This is a traditional name for YACC-like functionality + public abstract int yylex(); + + /// + /// Traditional error reporting provided by LEX-like scanners + /// to their YACC-like clients. + /// + /// Message format string + /// Optional array of args + [SuppressMessage( "Microsoft.Naming", "CA1709:IdentifiersShouldBeCasedCorrectly", MessageId = "yyerror" )] + [SuppressMessage( "Microsoft.Naming", "CA1704:IdentifiersShouldBeSpelledCorrectly", MessageId = "yyerror" )] + // Reason for FxCop message suppression - + // This is a traditional name for YACC-like functionality + public virtual void yyerror( string format, params object[] args ) { } + } + + /// + /// Encapsulated state for the parser. + /// Opaque to users, visible to the tool-generated code. + /// +#if EXPORT_GPPG + public class State + { + /// + /// The number of states in the automaton. + /// + public int number; +#else + internal class State { + /// + /// The index of this state in the states array. + /// + internal int number; +#endif + internal Dictionary ParserTable; // Terminal -> ParseAction + internal Dictionary Goto; // NonTerminal -> State; + internal int defaultAction; // = 0; // ParseAction + + /// + /// State transition data for this state. Pairs of elements of the + /// goto array associate symbol ordinals with next state indices. + /// The actions array is passed to another constructor. + /// + /// The action listc + /// Next state data + public State( int[] actions, int[] goToList ) + : this( actions ) { + Goto = new Dictionary(); + for (int i = 0; i < goToList.Length; i += 2) + Goto.Add( goToList[i], goToList[i + 1] ); + } + + /// + /// Action data for this state. Pairs of elements of the + /// action array associate action ordinals with each of + /// those symbols that have actions in the current state. + /// + /// The action array + public State( int[] actions ) { + ParserTable = new Dictionary(); + for (int i = 0; i < actions.Length; i += 2) + ParserTable.Add( actions[i], actions[i + 1] ); + } + + /// + /// Set the default action for this state. + /// + /// Ordinal of the default action + public State( int defaultAction ) { + this.defaultAction = defaultAction; + } + + /// + /// Set the default action and the state transition table. + /// + /// The default action + /// Transitions from this state + public State( int defaultAction, int[] goToList ) + : this( defaultAction ) { + Goto = new Dictionary(); + for (int i = 0; i < goToList.Length; i += 2) + Goto.Add( goToList[i], goToList[i + 1] ); + } + } + + /// + /// Rule representation at runtime. + /// +#if EXPORT_GPPG + public class Rule +#else + internal class Rule +#endif + { + internal int LeftHandSide; // symbol + internal int[] RightHandSide; // symbols + + /// + /// Rule constructor. This holds the ordinal of + /// the left hand side symbol, and the list of + /// right hand side symbols, in lexical order. + /// + /// The LHS non-terminal + /// The RHS symbols, in lexical order + public Rule( int left, int[] right ) { + this.LeftHandSide = left; + this.RightHandSide = right; + } + } + + /// + /// Stack utility for the shift-reduce parser. + /// GPPG parsers have three instances: + /// (1) The parser state stack, T = QUT.Gppg.State, + /// (2) The semantic value stack, T = TValue, + /// (3) The location stack, T = TSpan. + /// + /// +#if EXPORT_GPPG + public class PushdownPrefixState +#else + internal class PushdownPrefixState +#endif + { + // Note that we cannot use the BCL Stack class + // here as derived types need to index into stacks. + // + private T[] array = new T[8]; + private int tos = 0; + + /// + /// Indexer for values of the stack below the top. + /// + /// index of the element, starting from the bottom + /// the selected element + public T this[int index] { get { return array[index]; } } + + /// + /// The current depth of the stack. + /// + public int Depth { get { return tos; } } + + internal void Push( T value ) { + if (tos >= array.Length) { + T[] newarray = new T[array.Length * 2]; + System.Array.Copy( array, newarray, tos ); + array = newarray; + } + array[tos++] = value; + } + + internal T Pop() { + T rslt = array[--tos]; + array[tos] = default( T ); + return rslt; + } + + internal T TopElement() { return array[tos - 1]; } + + internal bool IsEmpty() { return tos == 0; } + } +} + diff --git a/src/DataStreamSuite/DataStream/DataStream.csproj b/src/DataStreamSuite/DataStream/DataStream.csproj index 99b284a22..dcd90c3e9 100644 --- a/src/DataStreamSuite/DataStream/DataStream.csproj +++ b/src/DataStreamSuite/DataStream/DataStream.csproj @@ -9,22 +9,8 @@ + - - - $(Externals)\bin\cstools\Tools.dll - - - - - - - - - - - - diff --git a/src/DataStreamSuite/DataStream/Primitives/Project/ProjectHelper.cs b/src/DataStreamSuite/DataStream/Primitives/Project/ProjectHelper.cs index ec40969b7..f012c95ad 100644 --- a/src/DataStreamSuite/DataStream/Primitives/Project/ProjectHelper.cs +++ b/src/DataStreamSuite/DataStream/Primitives/Project/ProjectHelper.cs @@ -1,6 +1,10 @@ +using Bts.ColumnMapExpressions; + using System; using System.Collections; -using BCG.Evaluator.ColumnMapExpression; +using System.Collections.Generic; +using System.Linq; +// using BCG.Evaluator.ColumnMapExpression; namespace BCG { @@ -28,7 +32,8 @@ namespace BCG /// original (unprojected) schema /// dictionary mapping unprojected names to their projected names, dropped cols should not be there /// table name of the new schema - internal ProjectHelper (IBlueTableSchema isch, IDictionary origToAlias, string tableName) + internal ProjectHelper ( + IBlueTableSchema isch, IDictionary origToAlias, string tableName) { ArrayList colMapAl = new ArrayList (), colInfoAl = new ArrayList (); @@ -36,12 +41,12 @@ namespace BCG int c = 0; ArrayList iColNames = new ArrayList (isch.columnNames); iColNames.Sort (); - foreach (string cn in iColNames) + foreach (string colName in iColNames) { - if (origToAlias.Contains (cn)) + if (origToAlias.TryGetValue(colName, out var colAlias)) { - ColumnInfo ci = isch[cn]; - colInfoAl.Add (new ColumnInfo (origToAlias[ci.colName] as string, + ColumnInfo ci = isch[colName]; + colInfoAl.Add (new ColumnInfo (colAlias, colInfoAl.Count, ci.colType) ); @@ -62,40 +67,28 @@ namespace BCG /// /// /// - static syntax myParser = - new syntax(); /// - private static IDictionary myGetDictionary (IBlueTableSchema isch, string exp) + private static IDictionary myGetDictionary (IBlueTableSchema isch, string exp) { - IDictionary origToAlias = new Hashtable (); + IDictionary origToAlias = new Dictionary(); ColumnMapExpression e; if (DataStreamLoadFileConfig.instance.useStaticParser) { - lock (myParser) - { - e = (ColumnMapExpression) - myParser.Parse(exp); - } + e = ColumnMapExpression.Parse(exp); } else { - syntax parser = new syntax(); - e = (ColumnMapExpression) - myParser.Parse(exp); - + // TODO (maybe): Actually use non-static parser (unclear what advantages it will have) + e = ColumnMapExpression.Parse(exp); } - if (e.isInclusive) - { - origToAlias = new Hashtable (e.columns); + if (e.IsInclusive) { + origToAlias = e.Columns.ToDictionary(); origToAlias["id"] = "id"; + } else { + origToAlias = isch.columnInfo.ToDictionary( + ci => ci.colName, ci => e.Columns.GetValueOrDefault(ci.colName) ?? ci.colName); } - else - foreach (ColumnInfo ci in isch.columnInfo) - if (e.columns[ci.colName] == null) - origToAlias[ci.colName] = ci.colName; - else if ((string)e.columns[ci.colName] != ci.colName) - origToAlias[ci.colName] = e.columns[ci.colName]; return origToAlias; } diff --git a/src/DataStreamSuite/DataStream/Primitives/Project/ProjectStream.cs b/src/DataStreamSuite/DataStream/Primitives/Project/ProjectStream.cs index 45ed7b822..71cdc2c23 100644 --- a/src/DataStreamSuite/DataStream/Primitives/Project/ProjectStream.cs +++ b/src/DataStreamSuite/DataStream/Primitives/Project/ProjectStream.cs @@ -3,14 +3,17 @@ namespace BCG { using Spark.Core.Collections; + using System; using System.Collections; + using System.Collections.Generic; + using System.Linq; public class ProjectStream : ViewStream { private ProjectHelper myHelper; private string myColExp, myTableName; private ViewStream myVs; - private IDictionary myDic; + private IDictionary myDic; private HighPerformanceTimer _timer = new(); /// /// @@ -44,9 +47,12 @@ namespace BCG { /// stream to subscribe to as the input stream /// dictionary mapping the aliased to the original public ProjectStream (ViewStream vs, IDictionary dic) + : this(vs, GenericDictionaryWrapper.Create(dic)) {} + + public ProjectStream (ViewStream vs, IEnumerable> dic) { myVs = vs; - myDic = ProjectHelper.invertDic (dic); + myDic = dic.ToDictionary(kv => kv.Value, kv => kv.Key); } protected override void myOnFirstSubscriber() diff --git a/src/DataStreamSuite/DataStreamCore/DataStreamCore.csproj b/src/DataStreamSuite/DataStreamCore/DataStreamCore.csproj index acc509193..c41a5190c 100644 --- a/src/DataStreamSuite/DataStreamCore/DataStreamCore.csproj +++ b/src/DataStreamSuite/DataStreamCore/DataStreamCore.csproj @@ -5,6 +5,10 @@ disable + + + + diff --git a/src/DataStreamSuite/DataStreamExpressions/Argument.cs b/src/DataStreamSuite/DataStreamExpressions/Argument.cs new file mode 100644 index 000000000..e1f9ad0d8 --- /dev/null +++ b/src/DataStreamSuite/DataStreamExpressions/Argument.cs @@ -0,0 +1,32 @@ +namespace Bts.DataStreamExpressions; + +public class Argument { + public Argument() { } + public Argument(string str) => String = str; + public Argument(bool? boolean) => Boolean = boolean; + public Argument(int? integer) => Integer = integer; + public Argument(double? real) => Real = real; + public Argument(DateTime? dateTime) => DateTime = dateTime; + public Argument(DBNull? @null) => Null = @null; + public Argument(StatementExpression? statementExp) => StatementExp = statementExp; + public Argument(QuotedView? quotedView) => QuotedView = quotedView; + + public string? String { get; } + public bool? Boolean { get; } + public int? Integer { get; } + public double? Real { get; } + public DateTime? DateTime { get; } + public DBNull? Null { get; } + public StatementExpression? StatementExp { get; } + public QuotedView? QuotedView { get; } + public object? Obj => + String ?? + Boolean ?? + Integer ?? + Real ?? + DateTime ?? + Null ?? + (object?)StatementExp ?? + QuotedView; + +} diff --git a/src/DataStreamSuite/DataStreamExpressions/Assignment.cs b/src/DataStreamSuite/DataStreamExpressions/Assignment.cs new file mode 100644 index 000000000..86590fad7 --- /dev/null +++ b/src/DataStreamSuite/DataStreamExpressions/Assignment.cs @@ -0,0 +1,27 @@ +namespace Bts.DataStreamExpressions; + +using BCG; + +class Assignment : StatementExpression { + readonly StatementExpression expression; + readonly string id; + + public Assignment(string identifier, StatementExpression st) { + id = identifier; + expression = st; + } + + public override string print(int startCol, IList activeBindings) { + string prelude = id + " = "; + string result = prelude + expression.print(startCol + prelude.Length, activeBindings); + if (activeBindings.IndexOf(id) == -1) + activeBindings.Add(id); + return result; + } + + public override ViewStream? myGetView(IDictionary env) { + var result = expression.getView(env); + env[id] = expression; + return result; + } +} diff --git a/src/DataStreamSuite/DataStreamExpressions/CSharpString.cs b/src/DataStreamSuite/DataStreamExpressions/CSharpString.cs new file mode 100644 index 000000000..02b6663b6 --- /dev/null +++ b/src/DataStreamSuite/DataStreamExpressions/CSharpString.cs @@ -0,0 +1,11 @@ +namespace Bts.DataStreamExpressions; + +class CSharpString { + public CSharpString(string? value = null) => + Value = value; + + public string? Value { get; } + + public override string? ToString() => + Value is null ? "(string)null" : $"\"{Value}\""; +} diff --git a/src/DataStreamSuite/DataStreamExpressions/DataStreamExpression.cs b/src/DataStreamSuite/DataStreamExpressions/DataStreamExpression.cs new file mode 100644 index 000000000..438bc04ea --- /dev/null +++ b/src/DataStreamSuite/DataStreamExpressions/DataStreamExpression.cs @@ -0,0 +1,40 @@ +namespace Bts.DataStreamExpressions; + +using BCG; + +using System.Text; + +public class DataStreamExpression { + public DataStreamExpression(ImmutableList statements) => + Statements = statements; + + public ImmutableList Statements { get; } + + public string print(int startPos, IList activeBindings) { + var sb = new StringBuilder(); + foreach (var st in Statements) { + sb.Append(new string(' ', startPos)); + sb.Append(st.print(startPos, activeBindings)); + sb.AppendLine(";"); + } + sb.AppendLine(); + return sb.ToString(); + } + + public string printView(int startPos) { + var sb = new StringBuilder(); + foreach (var st in Statements) { + sb.Append(new string(' ', startPos)); + sb.AppendLine(st.printView(startPos)); + } + sb.AppendLine(); + return sb.ToString(); + } + + public ViewStream getView(IDictionary env) { + for (int i = 0; i < Statements.Count - 1; i++) { + Statements[i].getView(env); + } + return Statements[^1].getView(env); + } +} diff --git a/src/DataStreamSuite/DataStreamExpressions/DataStreamExpressionGrammar.y b/src/DataStreamSuite/DataStreamExpressions/DataStreamExpressionGrammar.y new file mode 100644 index 000000000..7b719a52b --- /dev/null +++ b/src/DataStreamSuite/DataStreamExpressions/DataStreamExpressionGrammar.y @@ -0,0 +1,79 @@ +%namespace Bts.DataStreamExpressions +%partial +%visibility internal +%tokentype DataStreamExpressionToken +%parsertype DataStreamExpressionParser + +%union { + public int Integer; + public double Real; + public bool Boolean; + public string CsString; + public string Identifier; + public DateTime DateTime; + public DBNull Null; + public DataStreamExpression DataStream; + public StatementExpression Statement; + public PrimitiveExpression Primitive; + public ImmutableList Arguments; + public Argument Argument; + public string ViewName; + public QuotedView QuotedView; +} + +%token BOOLEAN_CONSTANT, INTEGER_CONSTANT, REAL_CONSTANT, DATE_TIME_CONSTANT, CS_STRING_CONSTANT, NULL_CONSTANT +%token PARENTHESIS_OPEN, PARENTHESIS_CLOSE, COMMA, IDENTIFIER, EQUALS, QUOTE_OPEN, QUOTE_CLOSE + +%nonassoc EQUALS + +%type DataStream +%type Statement +%type Argument +%type Arguments +%type QuotedView + +%% + +DataStream : + Statement { $$ = new(System.Collections.Immutable.ImmutableList.Empty.Add($1)); } + | Statement DataStream { $$ = new($2.Statements.Insert(0, $1)); } + ; + +Statement : + IDENTIFIER PARENTHESIS_OPEN Arguments PARENTHESIS_CLOSE { + $$ = new PrimitiveExpression($1.Identifier, $3); + } + | IDENTIFIER { + $$ = new ViewNameExpression($1.Identifier); + } + | IDENTIFIER EQUALS Statement { + $$ = new Assignment($1.Identifier, $3); + } + ; + +Arguments : + { + $$ = System.Collections.Immutable.ImmutableList.Empty; + } + | Argument { + $$ = System.Collections.Immutable.ImmutableList.Empty.Add($1); + } + | Arguments COMMA Argument { + $$ = $1.Add($3); + } + ; + +Argument : + CS_STRING_CONSTANT { $$ = new($1.CsString); } + | BOOLEAN_CONSTANT { $$ = new($1.Boolean); } + | INTEGER_CONSTANT { $$ = new($1.Integer); } + | REAL_CONSTANT { $$ = new($1.Real); } + | DATE_TIME_CONSTANT { $$ = new($1.DateTime); } + | NULL_CONSTANT { $$ = new($1.Null); } + | Statement { $$ = new($1); } + | QuotedView { $$ = new($1); } + ; + +QuotedView : + QUOTE_OPEN DataStream QUOTE_CLOSE { $$ = new($2.printView(0)); } + ; diff --git a/src/DataStreamSuite/DataStreamExpressions/DataStreamExpressionInterpreter.cs b/src/DataStreamSuite/DataStreamExpressions/DataStreamExpressionInterpreter.cs new file mode 100644 index 000000000..0ab2c3cf8 --- /dev/null +++ b/src/DataStreamSuite/DataStreamExpressions/DataStreamExpressionInterpreter.cs @@ -0,0 +1,19 @@ +namespace Bts.DataStreamExpressions; + +using System.IO; +using System.Text; + +public class DataStreamExpressionsInterpreter { + public static DataStreamExpression Evaluate(Stream stream) { + var parser = new DataStreamExpressionParser(); + var res = parser.Parse(stream); + var program = res?.DataStream ?? throw new Exception( + "Failed to parse boolean expression"); + return program; + } + + public static DataStreamExpression Evaluate(string str) { + using var ms = new MemoryStream(Encoding.Default.GetBytes(str)); + return Evaluate(ms); + } +} diff --git a/src/DataStreamSuite/DataStreamExpressions/DataStreamExpressionParser.cs b/src/DataStreamSuite/DataStreamExpressions/DataStreamExpressionParser.cs new file mode 100644 index 000000000..3282586cc --- /dev/null +++ b/src/DataStreamSuite/DataStreamExpressions/DataStreamExpressionParser.cs @@ -0,0 +1,18 @@ +namespace Bts.DataStreamExpressions; + +using System.Text; + +partial class DataStreamExpressionParser { + public DataStreamExpressionParser() : base(null) { } + + public ValueType? Parse(string s) { + using var ms = new MemoryStream(Encoding.Default.GetBytes(s)); + return Parse(ms); + } + + public ValueType? Parse(Stream stream) { + Scanner = new DataStreamExpressionScanner(stream); + Parse(); + return CurrentSemanticValue; + } +} diff --git a/src/DataStreamSuite/DataStreamExpressions/DataStreamExpressionScanner.cs b/src/DataStreamSuite/DataStreamExpressions/DataStreamExpressionScanner.cs new file mode 100644 index 000000000..c2fb8f3fa --- /dev/null +++ b/src/DataStreamSuite/DataStreamExpressions/DataStreamExpressionScanner.cs @@ -0,0 +1,9 @@ +namespace Bts.DataStreamExpressions; + +partial class DataStreamExpressionScanner { + public override void yyerror(string format, params object[] args) { + base.yyerror(format, args); + Console.WriteLine(format, args); + Console.WriteLine(); + } +} diff --git a/src/DataStreamSuite/DataStreamExpressions/DataStreamExpressionTokens.lex b/src/DataStreamSuite/DataStreamExpressions/DataStreamExpressionTokens.lex new file mode 100644 index 000000000..42c789517 --- /dev/null +++ b/src/DataStreamSuite/DataStreamExpressions/DataStreamExpressionTokens.lex @@ -0,0 +1,95 @@ +%namespace Bts.DataStreamExpressions +%scannertype DataStreamExpressionScanner +%visibility internal +%tokentype DataStreamExpressionToken + +%option stack, minimize, parser, verbose, persistbuffer, noembedbuffers + +/* Literal Constants */ +Letter [a-zA-Z] +BooleanConstant 'true'|'false' +CSStringConstant \"([^\"\\]*|\\.)*\" +DateTimeConstant ([1-9]|'10'|'11'|'12')\/([1-9]|([1-2][0-9])|'30'|'31')\/[0-9][0-9][0-9]* +IntegerConstant -?[0-9]+ +NullConstant 'null' +RealConstant -?[0-9]+\.[0-9]+ + +/* Relational Operators */ +Equals = + +Identifier {Letter}({Letter}*[0-9]*_*)* + +/* Utility Tokens */ +OpenParenthesis \( +CloseParenthesis \) +Comma , +QuoteOpen \[ +QuoteClose \] + +%% + +{Equals} { + return (int)DataStreamExpressionToken.EQUALS; +} + +{CSStringConstant} { + yylval.CsString = yytext.Substring(1, yytext.Length - 2); + return (int)DataStreamExpressionToken.CS_STRING_CONSTANT; +} + +{BooleanConstant} { + yylval.Boolean = yytext.ToLower() switch { + "true" => true, + "false" => false, + _ => throw new Exception( + string.Format("Expected yytext to be 'true' or 'false' (actually '{0}')", yytext)) + }; + return (int)DataStreamExpressionToken.BOOLEAN_CONSTANT; +} + +{NullConstant} { + yylval.Null = DBNull.Value; + return (int)DataStreamExpressionToken.NULL_CONSTANT; +} + +{DateTimeConstant} { + yylval.DateTime = DateTime.Parse(yytext); + return (int)DataStreamExpressionToken.DATE_TIME_CONSTANT; +} + +{IntegerConstant} { + yylval.Integer = int.Parse(yytext); + return (int)DataStreamExpressionToken.INTEGER_CONSTANT; +} + +{RealConstant} { + yylval.Real = double.Parse(yytext); + return (int)DataStreamExpressionToken.REAL_CONSTANT; +} + +{OpenParenthesis} { + return (int)DataStreamExpressionToken.PARENTHESIS_OPEN; +} + +{CloseParenthesis} { + return (int)DataStreamExpressionToken.PARENTHESIS_CLOSE; +} + +{Comma} { + return (int)DataStreamExpressionToken.COMMA; +} + +{QuoteOpen} { + return (int)DataStreamExpressionToken.QUOTE_CLOSE; +} + +{QuoteClose} { + return (int)DataStreamExpressionToken.QUOTE_OPEN; +} + +{Identifier} { + yylval.Identifier = yytext; + return (int)DataStreamExpressionToken.IDENTIFIER; +} + +%% diff --git a/src/DataStreamSuite/DataStreamExpressions/DataStreamExpressions.csproj b/src/DataStreamSuite/DataStreamExpressions/DataStreamExpressions.csproj new file mode 100644 index 000000000..20f40bbbb --- /dev/null +++ b/src/DataStreamSuite/DataStreamExpressions/DataStreamExpressions.csproj @@ -0,0 +1,26 @@ + + + + net7.0 + enable + enable + + + + + + + + + + + + + + + + + + + + diff --git a/src/DataStreamSuite/DataStreamExpressions/GplexBuffers.cs b/src/DataStreamSuite/DataStreamExpressions/GplexBuffers.cs new file mode 100644 index 000000000..76dd35443 --- /dev/null +++ b/src/DataStreamSuite/DataStreamExpressions/GplexBuffers.cs @@ -0,0 +1,720 @@ +// ============================================================== +// +// This code automatically produced from an embedded resource. +// Do not edit this file, or it will become incompatible with +// the specification from which it was generated. +// +// ============================================================== + +using System; +using System.IO; +using System.Text; +using System.Collections.Generic; +using System.Diagnostics.CodeAnalysis; +using System.Runtime.Serialization; +using System.Globalization; + +namespace QUT.GplexBuffers +{ +// Code copied from GPLEX embedded resource + [Serializable] + internal class BufferException : Exception + { + public BufferException() { } + public BufferException(string message) : base(message) { } + public BufferException(string message, Exception innerException) + : base(message, innerException) { } + protected BufferException(SerializationInfo info, StreamingContext context) + : base(info, context) { } + } + + internal abstract class ScanBuff + { + private string fileNm; + + public const int EndOfFile = -1; + public const int UnicodeReplacementChar = 0xFFFD; + + public bool IsFile { get { return (fileNm != null); } } + public string FileName { get { return fileNm; } set { fileNm = value; } } + + public abstract int Pos { get; set; } + public abstract int Read(); + public virtual void Mark() { } + + public abstract string GetString(int begin, int limit); + + public static ScanBuff GetBuffer(string source) + { + return new StringBuffer(source); + } + + public static ScanBuff GetBuffer(IList source) + { + return new LineBuffer(source); + } + +#if (!NOFILES) + public static ScanBuff GetBuffer(Stream source) + { + return new BuildBuffer(source); + } + +#if (!BYTEMODE) + public static ScanBuff GetBuffer(Stream source, int fallbackCodePage) + { + return new BuildBuffer(source, fallbackCodePage); + } +#endif // !BYTEMODE +#endif // !NOFILES + } + + #region Buffer classes + + // ============================================================== + // ===== Definitions for various ScanBuff derived classes ==== + // ============================================================== + // =============== String input ================ + // ============================================================== + + /// + /// This class reads characters from a single string as + /// required, for example, by Visual Studio language services + /// + sealed class StringBuffer : ScanBuff + { + string str; // input buffer + int bPos; // current position in buffer + int sLen; + + public StringBuffer(string source) + { + this.str = source; + this.sLen = source.Length; + this.FileName = null; + } + + public override int Read() + { + if (bPos < sLen) return str[bPos++]; + else if (bPos == sLen) { bPos++; return '\n'; } // one strike, see new line + else { bPos++; return EndOfFile; } // two strikes and you're out! + } + + public override string GetString(int begin, int limit) + { + // "limit" can be greater than sLen with the BABEL + // option set. Read returns a "virtual" EOL if + // an attempt is made to read past the end of the + // string buffer. Without the guard any attempt + // to fetch yytext for a token that includes the + // EOL will throw an index exception. + if (limit > sLen) limit = sLen; + if (limit <= begin) return ""; + else return str.Substring(begin, limit - begin); + } + + public override int Pos + { + get { return bPos; } + set { bPos = value; } + } + + public override string ToString() { return "StringBuffer"; } + } + + // ============================================================== + // The LineBuff class contributed by Nigel Horspool, + // nigelh@cs.uvic.cs + // ============================================================== + + sealed class LineBuffer : ScanBuff + { + IList line; // list of source lines from a file + int numLines; // number of strings in line list + string curLine; // current line in that list + int cLine; // index of current line in the list + int curLen; // length of current line + int curLineStart; // position of line start in whole file + int curLineEnd; // position of line end in whole file + int maxPos; // max position ever visited in whole file + int cPos; // ordinal number of code in source + + // Constructed from a list of strings, one per source line. + // The lines have had trailing '\n' characters removed. + public LineBuffer(IList lineList) + { + line = lineList; + numLines = line.Count; + cPos = curLineStart = 0; + curLine = (numLines > 0 ? line[0] : ""); + maxPos = curLineEnd = curLen = curLine.Length; + cLine = 1; + FileName = null; + } + + public override int Read() + { + if (cPos < curLineEnd) + return curLine[cPos++ - curLineStart]; + if (cPos++ == curLineEnd) + return '\n'; + if (cLine >= numLines) + return EndOfFile; + curLine = line[cLine]; + curLen = curLine.Length; + curLineStart = curLineEnd + 1; + curLineEnd = curLineStart + curLen; + if (curLineEnd > maxPos) + maxPos = curLineEnd; + cLine++; + return curLen > 0 ? curLine[0] : '\n'; + } + + // To speed up searches for the line containing a position + private int cachedPosition; + private int cachedIxdex; + private int cachedLineStart; + + // Given a position pos within the entire source, the results are + // ix -- the index of the containing line + // lstart -- the position of the first character on that line + private void findIndex(int pos, out int ix, out int lstart) + { + if (pos >= cachedPosition) + { + ix = cachedIxdex; lstart = cachedLineStart; + } + else + { + ix = lstart = 0; + } + while (ix < numLines) + { + int len = line[ix].Length + 1; + if (pos < lstart + len) break; + lstart += len; + ix++; + } + cachedPosition = pos; + cachedIxdex = ix; + cachedLineStart = lstart; + } + + public override string GetString(int begin, int limit) + { + if (begin >= maxPos || limit <= begin) return ""; + int endIx, begIx, endLineStart, begLineStart; + findIndex(begin, out begIx, out begLineStart); + int begCol = begin - begLineStart; + findIndex(limit, out endIx, out endLineStart); + int endCol = limit - endLineStart; + string s = line[begIx]; + if (begIx == endIx) + { + // the usual case, substring all on one line + return (endCol <= s.Length) ? + s.Substring(begCol, endCol - begCol) + : s.Substring(begCol) + "\n"; + } + // the string spans multiple lines, yuk! + StringBuilder sb = new StringBuilder(); + if (begCol < s.Length) + sb.Append(s.Substring(begCol)); + for (; ; ) + { + sb.Append("\n"); + s = line[++begIx]; + if (begIx >= endIx) break; + sb.Append(s); + } + if (endCol <= s.Length) + { + sb.Append(s.Substring(0, endCol)); + } + else + { + sb.Append(s); + sb.Append("\n"); + } + return sb.ToString(); + } + + public override int Pos + { + get { return cPos; } + set + { + cPos = value; + findIndex(cPos, out cLine, out curLineStart); + // cLine should be the *next* line after curLine. + curLine = (cLine < numLines ? line[cLine++] : ""); + curLineEnd = curLineStart + curLine.Length; + } + } + + public override string ToString() { return "LineBuffer"; } + } + +#if (!NOFILES) + // ============================================================== + // ===== class BuildBuff : for unicode text files ======== + // ============================================================== + + class BuildBuffer : ScanBuff + { + // Double buffer for char stream. + class BufferElement + { + StringBuilder bldr = new StringBuilder(); + StringBuilder next = new StringBuilder(); + int minIx; + int maxIx; + int brkIx; + bool appendToNext; + + internal BufferElement() { } + + internal int MaxIndex { get { return maxIx; } } + // internal int MinIndex { get { return minIx; } } + + internal char this[int index] + { + get + { + if (index < minIx || index >= maxIx) + throw new BufferException("Index was outside data buffer"); + else if (index < brkIx) + return bldr[index - minIx]; + else + return next[index - brkIx]; + } + } + + internal void Append(char[] block, int count) + { + maxIx += count; + if (appendToNext) + this.next.Append(block, 0, count); + else + { + this.bldr.Append(block, 0, count); + brkIx = maxIx; + appendToNext = true; + } + } + + internal string GetString(int start, int limit) + { + if (limit <= start) + return ""; + if (start >= minIx && limit <= maxIx) + if (limit < brkIx) // String entirely in bldr builder + return bldr.ToString(start - minIx, limit - start); + else if (start >= brkIx) // String entirely in next builder + return next.ToString(start - brkIx, limit - start); + else // Must do a string-concatenation + return + bldr.ToString(start - minIx, brkIx - start) + + next.ToString(0, limit - brkIx); + else + throw new BufferException("String was outside data buffer"); + } + + internal void Mark(int limit) + { + if (limit > brkIx + 16) // Rotate blocks + { + StringBuilder temp = bldr; + bldr = next; + next = temp; + next.Length = 0; + minIx = brkIx; + brkIx = maxIx; + } + } + } + + BufferElement data = new BufferElement(); + + int bPos; // Postion index in the StringBuilder + BlockReader NextBlk; // Delegate that serves char-arrays; + + private string EncodingName + { + get + { + StreamReader rdr = NextBlk.Target as StreamReader; + return (rdr == null ? "raw-bytes" : rdr.CurrentEncoding.BodyName); + } + } + + public BuildBuffer(Stream stream) + { + FileStream fStrm = (stream as FileStream); + if (fStrm != null) FileName = fStrm.Name; + NextBlk = BlockReaderFactory.Raw(stream); + } + +#if (!BYTEMODE) + public BuildBuffer(Stream stream, int fallbackCodePage) + { + FileStream fStrm = (stream as FileStream); + if (fStrm != null) FileName = fStrm.Name; + NextBlk = BlockReaderFactory.Get(stream, fallbackCodePage); + } +#endif + + /// + /// Marks a conservative lower bound for the buffer, + /// allowing space to be reclaimed. If an application + /// needs to call GetString at arbitrary past locations + /// in the input stream, Mark() is not called. + /// + public override void Mark() { data.Mark(bPos - 2); } + + public override int Pos + { + get { return bPos; } + set { bPos = value; } + } + + + /// + /// Read returns the ordinal number of the next char, or + /// EOF (-1) for an end of stream. Note that the next + /// code point may require *two* calls of Read(). + /// + /// + public override int Read() + { + // + // Characters at positions + // [data.offset, data.offset + data.bldr.Length) + // are available in data.bldr. + // + if (bPos < data.MaxIndex) + { + // ch0 cannot be EOF + return (int)data[bPos++]; + } + else // Read from underlying stream + { + // Experimental code, blocks of page size + char[] chrs = new char[4096]; + int count = NextBlk(chrs, 0, 4096); + if (count == 0) + return EndOfFile; + else + { + data.Append(chrs, count); + return (int)data[bPos++]; + } + } + } + + public override string GetString(int begin, int limit) + { + return data.GetString(begin, limit); + } + + public override string ToString() + { + return "StringBuilder buffer, encoding: " + this.EncodingName; + } + } + + // =============== End ScanBuff-derived classes ================== + + internal delegate int BlockReader(char[] block, int index, int number); + + // A delegate factory, serving up a delegate that + // reads a block of characters from the underlying + // encoded stream, via a StreamReader object. + // + internal static class BlockReaderFactory + { + public static BlockReader Raw(Stream stream) + { + return delegate(char[] block, int index, int number) + { + byte[] b = new byte[number]; + int count = stream.Read(b, 0, number); + int i = 0; + int j = index; + for (; i < count; i++, j++) + block[j] = (char)b[i]; + return count; + }; + } + +#if (!BYTEMODE) + public static BlockReader Get(Stream stream, int fallbackCodePage) + { + Encoding encoding; + int preamble = Preamble(stream); + + if (preamble != 0) // There is a valid BOM here! + encoding = Encoding.GetEncoding(preamble); + else if (fallbackCodePage == -1) // Fallback is "raw" bytes + return Raw(stream); + else if (fallbackCodePage != -2) // Anything but "guess" + encoding = Encoding.GetEncoding(fallbackCodePage); + else // This is the "guess" option + { + int guess = new Guesser(stream).GuessCodePage(); + stream.Seek(0, SeekOrigin.Begin); + if (guess == -1) // ==> this is a 7-bit file + encoding = Encoding.ASCII; + else if (guess == 65001) + encoding = Encoding.UTF8; + else // ==> use the machine default + encoding = Encoding.Default; + } + StreamReader reader = new StreamReader(stream, encoding); + return reader.Read; + } + + static int Preamble(Stream stream) + { + int b0 = stream.ReadByte(); + int b1 = stream.ReadByte(); + + if (b0 == 0xfe && b1 == 0xff) + return 1201; // UTF16BE + if (b0 == 0xff && b1 == 0xfe) + return 1200; // UTF16LE + + int b2 = stream.ReadByte(); + if (b0 == 0xef && b1 == 0xbb && b2 == 0xbf) + return 65001; // UTF8 + // + // There is no unicode preamble, so we + // return denoter for the machine default. + // + stream.Seek(0, SeekOrigin.Begin); + return 0; + } +#endif // !BYTEMODE + } +#endif // !NOFILES + #endregion Buffer classes + + // ============================================================== + // ============ class CodePageHandling ============= + // ============================================================== +#if (!NOFILES) + public static class CodePageHandling + { + public static int GetCodePage(string option) + { + string command = option.ToUpperInvariant(); + if (command.StartsWith("CodePage:", StringComparison.OrdinalIgnoreCase)) + command = command.Substring(9); + try + { + if (command.Equals("RAW")) + return -1; + else if (command.Equals("GUESS")) + return -2; + else if (command.Equals("DEFAULT")) + return 0; + else if (char.IsDigit(command[0])) + return int.Parse(command, CultureInfo.InvariantCulture); + else + { + Encoding enc = Encoding.GetEncoding(command); + return enc.CodePage; + } + } + catch (FormatException) + { + Console.Error.WriteLine( + "Invalid format \"{0}\", using machine default", option); + } + catch (ArgumentException) + { + Console.Error.WriteLine( + "Unknown code page \"{0}\", using machine default", option); + } + return 0; + } + } +#region guesser +#if (!BYTEMODE) + // ============================================================== + // ============ Encoding Guesser ============= + // ============================================================== + + /// + /// This class provides a simple finite state automaton that + /// scans the file looking for (1) valid UTF-8 byte patterns, + /// (2) bytes >= 0x80 which are not part of a UTF-8 sequence. + /// The method then guesses whether it is UTF-8 or maybe some + /// local machine default encoding. This works well for the + /// various Latin encodings. + /// + internal class Guesser + { + ScanBuff buffer; + + public int GuessCodePage() { return Scan(); } + + const int maxAccept = 10; + const int initial = 0; + const int eofNum = 0; + const int goStart = -1; + const int INITIAL = 0; + const int EndToken = 0; + + #region user code + /* + * Reads the bytes of a file to determine if it is + * UTF-8 or a single-byte code page file. + */ + public long utfX; + public long uppr; + #endregion user code + + int state; + int currentStart = startState[0]; + int code; + + #region ScannerTables + static int[] startState = new int[] { 11, 0 }; + + #region CharacterMap + static sbyte[] map = new sbyte[256] { +/* '\0' */ 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, +/* '\x10' */ 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, +/* '\x20' */ 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, +/* '0' */ 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, +/* '@' */ 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, +/* 'P' */ 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, +/* '`' */ 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, +/* 'p' */ 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, +/* '\x80' */ 2, 2, 2, 2, 2, 2, 2, 2, 2, 2, 2, 2, 2, 2, 2, 2, +/* '\x90' */ 2, 2, 2, 2, 2, 2, 2, 2, 2, 2, 2, 2, 2, 2, 2, 2, +/* '\xA0' */ 2, 2, 2, 2, 2, 2, 2, 2, 2, 2, 2, 2, 2, 2, 2, 2, +/* '\xB0' */ 2, 2, 2, 2, 2, 2, 2, 2, 2, 2, 2, 2, 2, 2, 2, 2, +/* '\xC0' */ 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, +/* '\xD0' */ 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, +/* '\xE0' */ 3, 3, 3, 3, 3, 3, 3, 3, 3, 3, 3, 3, 3, 3, 3, 3, +/* '\xF0' */ 4, 4, 4, 4, 4, 4, 4, 4, 5, 5, 5, 5, 5, 5, 5, 5 }; + #endregion + + static sbyte[][] nextState = new sbyte[][] { + new sbyte[] {0, 0, 0, 0, 0, 0}, + new sbyte[] {-1, -1, 10, -1, -1, -1}, + new sbyte[] {-1, -1, -1, -1, -1, -1}, + new sbyte[] {-1, -1, 8, -1, -1, -1}, + new sbyte[] {-1, -1, 5, -1, -1, -1}, + new sbyte[] {-1, -1, 6, -1, -1, -1}, + new sbyte[] {-1, -1, 7, -1, -1, -1}, + null, + new sbyte[] {-1, -1, 9, -1, -1, -1}, + null, + null, + new sbyte[] {-1, 1, 2, 3, 4, 2} + }; + + + [SuppressMessage("Microsoft.Performance", "CA1810:InitializeReferenceTypeStaticFieldsInline")] + // Reason for suppression: cannot have self-reference in array initializer. + static Guesser() + { + nextState[7] = nextState[2]; + nextState[9] = nextState[2]; + nextState[10] = nextState[2]; + } + + int NextState() + { + if (code == ScanBuff.EndOfFile) + return eofNum; + else + return nextState[state][map[code]]; + } + #endregion + + public Guesser(System.IO.Stream file) { SetSource(file); } + + public void SetSource(System.IO.Stream source) + { + this.buffer = new BuildBuffer(source); + code = buffer.Read(); + } + + int Scan() + { + for (; ; ) + { + int next; + state = currentStart; + while ((next = NextState()) == goStart) + code = buffer.Read(); + + state = next; + code = buffer.Read(); + + while ((next = NextState()) > eofNum) + { + state = next; + code = buffer.Read(); + } + if (state <= maxAccept) + { + #region ActionSwitch +#pragma warning disable 162 + switch (state) + { + case eofNum: + switch (currentStart) + { + case 11: + if (utfX == 0 && uppr == 0) return -1; /* raw ascii */ + else if (uppr * 10 > utfX) return 0; /* default code page */ + else return 65001; /* UTF-8 encoding */ + break; + } + return EndToken; + case 1: // Recognized '{Upper128}', Shortest string "\xC0" + case 2: // Recognized '{Upper128}', Shortest string "\x80" + case 3: // Recognized '{Upper128}', Shortest string "\xE0" + case 4: // Recognized '{Upper128}', Shortest string "\xF0" + uppr++; + break; + case 5: // Recognized '{Utf8pfx4}{Utf8cont}', Shortest string "\xF0\x80" + uppr += 2; + break; + case 6: // Recognized '{Utf8pfx4}{Utf8cont}{2}', Shortest string "\xF0\x80\x80" + uppr += 3; + break; + case 7: // Recognized '{Utf8pfx4}{Utf8cont}{3}', Shortest string "\xF0\x80\x80\x80" + utfX += 3; + break; + case 8: // Recognized '{Utf8pfx3}{Utf8cont}', Shortest string "\xE0\x80" + uppr += 2; + break; + case 9: // Recognized '{Utf8pfx3}{Utf8cont}{2}', Shortest string "\xE0\x80\x80" + utfX += 2; + break; + case 10: // Recognized '{Utf8pfx2}{Utf8cont}', Shortest string "\xC0\x80" + utfX++; + break; + default: + break; + } +#pragma warning restore 162 + #endregion + } + } + } + } // end class Guesser + +#endif // !BYTEMODE +#endregion +#endif // !NOFILES + +// End of code copied from embedded resource +} diff --git a/src/DataStreamSuite/DataStreamExpressions/IArgument.cs b/src/DataStreamSuite/DataStreamExpressions/IArgument.cs new file mode 100644 index 000000000..e69de29bb diff --git a/src/DataStreamSuite/DataStreamExpressions/PrimitiveExpression.cs b/src/DataStreamSuite/DataStreamExpressions/PrimitiveExpression.cs new file mode 100644 index 000000000..045157b83 --- /dev/null +++ b/src/DataStreamSuite/DataStreamExpressions/PrimitiveExpression.cs @@ -0,0 +1,112 @@ +namespace Bts.DataStreamExpressions; + +using BCG; + +using System.Collections; +using System.Collections.Immutable; + +public class PrimitiveExpression : StatementExpression { + private string myName; + IList myArgs; + + public PrimitiveExpression(string name) { + myName = name; + myArgs = new List(0); + } + + public PrimitiveExpression(string name, ImmutableList args) { + myName = name; + myArgs = args; + } + + public override string print(int startCol, IList activeBindings) { + string prelude = "new " + PrimitiveManager.instance.find(myName) + " ("; + + return prelude + myPrintArgs(startCol + prelude.Length, activeBindings) + ")"; + } + + public override string printView(int startCol) { + string prelude = myName + " ("; + + return prelude + myPrintViewArgs(startCol + prelude.Length) + ")"; + } + + public override ViewStream myGetView(IDictionary env) { + ArrayList resolvedArgs = new ArrayList(); + + foreach (var arg in myArgs) { + if (arg.StatementExp is not null) { + resolvedArgs.Add(arg.StatementExp.myGetView(env)); + continue; + } + if (arg.QuotedView is not null) { + resolvedArgs.Add(arg.QuotedView.Value); + continue; + } + resolvedArgs.Add(arg.Obj); + } + return PrimitiveManager.instance.build(myName, resolvedArgs); + } + + + string myPrintSingleArg(int atColumn, object o, IList activeBindings) { + if (o is StatementExpression) + return new string(' ', atColumn) + + ((o as StatementExpression)?.print(atColumn, activeBindings) ?? string.Empty); + else { + string result = new string(' ', atColumn); + + if (o is string) + result += makeCSString((string)o); + else if (o is DateTime) + result += "DateTime.parse(\"" + ((DateTime)o).ToString() + "\")"; + else if (o is QuotedView) + result += "\"" + ((QuotedView)o).Value + "\""; + else if (o is bool) + result += o.Equals(true) ? "true" : "false"; + else + result += o.ToString(); + + return result; + } + } + + string myPrintArgs(int atColumn, IList activeBindings) { + string result = ""; + for (int i = 0; i < myArgs.Count; i++) + if (i == 0) + result += myPrintSingleArg(0, myArgs[i], activeBindings); + else + result += ",\n" + myPrintSingleArg(atColumn, myArgs[i], activeBindings); + return result; + } + + string myPrintViewArgs(int atColumn) { + string result = ""; + for (int i = 0; i < myArgs.Count; i++) + if (i == 0) + result += myPrintSingleViewArg(0, myArgs[i]); + else + result += ",\n" + myPrintSingleViewArg(atColumn, myArgs[i]); + return result; + } + + string myPrintSingleViewArg(int atColumn, object o) { + if (o is StatementExpression) + return new string(' ', atColumn) + (o as StatementExpression).printView(atColumn); + else { + string result = new string(' ', atColumn); + + if (o is string) + result += new CSharpString((string)o).ToString(); + else if (o is QuotedView) + result += "[" + ((QuotedView)o).Value + "]"; + else if (o is bool) + result += o.Equals(true) ? "true" : "false"; + else + result += o.ToString(); + + return result; + } + } +} diff --git a/src/DataStreamSuite/DataStreamExpressions/QuotedView.cs b/src/DataStreamSuite/DataStreamExpressions/QuotedView.cs new file mode 100644 index 000000000..95937ba12 --- /dev/null +++ b/src/DataStreamSuite/DataStreamExpressions/QuotedView.cs @@ -0,0 +1,8 @@ +namespace Bts.DataStreamExpressions; + +public class QuotedView { + public QuotedView(string value) => + Value = value; + + public string Value { get; } +} diff --git a/src/DataStreamSuite/DataStreamExpressions/ShiftReduceParserCode.cs b/src/DataStreamSuite/DataStreamExpressions/ShiftReduceParserCode.cs new file mode 100644 index 000000000..a255acb0d --- /dev/null +++ b/src/DataStreamSuite/DataStreamExpressions/ShiftReduceParserCode.cs @@ -0,0 +1,884 @@ +// ============================================================== +// +// This code automatically produced from an embedded resource. +// Do not edit this file, or it will become incompatible with +// the specification from which it was generated. +// +// ============================================================== + +// Gardens Point Parser Generator +// Copyright (c) Wayne Kelly, QUT 2005-2010 +// (see accompanying GPPGcopyright.rtf) + +using System; +using System.Text; +using System.Globalization; +using System.Collections.Generic; +using System.Runtime.Serialization; +using System.Diagnostics.CodeAnalysis; + +namespace QUT.Gppg { + /// + /// Abstract class for GPPG shift-reduce parsers. + /// Parsers generated by GPPG derive from this base + /// class, overriding the abstract Initialize() and + /// DoAction() methods. + /// + /// Semantic value type + /// Location type +#if EXPORT_GPPG + public abstract class ShiftReduceParser +#else + internal abstract class ShiftReduceParser +#endif + where TSpan : IMerge, new() { + private AbstractScanner scanner; + /// + /// The abstract scanner for this parser. + /// + protected AbstractScanner Scanner { + get { return scanner; } + set { scanner = value; } + } + + /// + /// Constructor for base class + /// + /// Scanner instance for this parser + protected ShiftReduceParser( AbstractScanner scanner ) { + this.scanner = scanner; + } + + // ============================================================== + // TECHNICAL EXPLANATION. + // Why the next two fields are not exposed via properties. + // ============================================================== + // These fields are of the generic parameter types, and are + // frequently instantiated as struct types in derived classes. + // Semantic actions are defined in the derived classes and refer + // to instance fields of these structs. In such cases the code + // "get_CurrentSemanticValue().myField = blah;" will fail since + // the getter pushes the value of the field, not the reference. + // So, in the presence of properties, gppg would need to encode + // such field accesses as ... + // "tmp = get_CurrentSemanticValue(); // Fetch value + // tmp.myField = blah; // update + // set_CurrentSemanticValue(tmp); " // Write update back. + // There is no issue if TValue is restricted to be a ref type. + // The same explanation applies to scanner.yylval. + // ============================================================== + /// + /// The current value of the "$$" symbolic variable in the parser + /// + [SuppressMessage( "Microsoft.Design", "CA1051:DoNotDeclareVisibleInstanceFields" )] + protected TValue CurrentSemanticValue; + + /// + /// The current value of the "@$" symbolic variable in the parser + /// + [SuppressMessage( "Microsoft.Design", "CA1051:DoNotDeclareVisibleInstanceFields" )] + protected TSpan CurrentLocationSpan; + protected int NextToken; + + private TSpan LastSpan; + private State FsaState; + private bool recovering; + private int tokensSinceLastError; + + private PushdownPrefixState StateStack = new PushdownPrefixState(); + private PushdownPrefixState valueStack = new PushdownPrefixState(); + private PushdownPrefixState locationStack = new PushdownPrefixState(); + + /// + /// The stack of semantic value (YYSTYPE) values. + /// + protected PushdownPrefixState ValueStack { get { return valueStack; } } + + /// + /// The stack of location value (YYLTYPE) varlues. + /// + protected PushdownPrefixState LocationStack { get { return locationStack; } } + + private int errorToken; + private int endOfFileToken; + private string[] nonTerminals; + private State[] states; + private Rule[] rules; + + /// + /// Initialization method to allow derived classes + /// to insert the rule list into this base class. + /// + /// The array of Rule objects + protected void InitRules( Rule[] rules ) { this.rules = rules; } + + /// + /// Initialization method to allow derived classes to + /// insert the states table into this base class. + /// + /// The pre-initialized states table + protected void InitStates( State[] states ) { this.states = states; } + + /// + /// OBSOLETE FOR VERSION 1.4.0 + /// + /// + protected void InitStateTable( int size ) { states = new State[size]; } + + /// + /// Initialization method to allow derived classes + /// to insert the special value for the error and EOF tokens. + /// + /// The error state ordinal + /// The EOF stat ordinal + protected void InitSpecialTokens( int err, int end ) { + errorToken = err; + endOfFileToken = end; + } + + /// + /// Initialization method to allow derived classes to + /// insert the non-terminal symbol names into this base class. + /// + /// Non-terminal symbol names + protected void InitNonTerminals( string[] names ) { nonTerminals = names; } + + #region YYAbort, YYAccept etcetera. + [Serializable] + [SuppressMessage( "Microsoft.Design", "CA1064:ExceptionsShouldBePublic" )] + // Reason for FxCop message suppression - + // This exception cannot escape from the local context + private class AcceptException : Exception { + internal AcceptException() { } + protected AcceptException( SerializationInfo i, StreamingContext c ) : base( i, c ) { } + } + [Serializable] + [SuppressMessage( "Microsoft.Design", "CA1064:ExceptionsShouldBePublic" )] + // Reason for FxCop message suppression - + // This exception cannot escape from the local context + private class AbortException : Exception { + internal AbortException() { } + protected AbortException( SerializationInfo i, StreamingContext c ) : base( i, c ) { } + } + [Serializable] + [SuppressMessage( "Microsoft.Design", "CA1064:ExceptionsShouldBePublic" )] + // Reason for FxCop message suppression - + // This exception cannot escape from the local context + private class ErrorException : Exception { + internal ErrorException() { } + protected ErrorException( SerializationInfo i, StreamingContext c ) : base( i, c ) { } + } + + // The following methods are only called from within + // a semantic action. The thrown exceptions can never + // propagate outside the ShiftReduceParser class in + // which they are nested. + + /// + /// Force parser to terminate, returning "true" + /// + protected static void YYAccept() { throw new AcceptException(); } + + /// + /// Force parser to terminate, returning "false" + /// + protected static void YYAbort() { throw new AbortException(); } + + /// + /// Force parser to terminate, returning + /// "false" if error recovery fails. + /// + protected static void YYError() { throw new ErrorException(); } + + /// + /// Check if parser in error recovery state. + /// + protected bool YYRecovering { get { return recovering; } } + #endregion + + /// + /// Abstract base method. ShiftReduceParser calls this + /// to initialize the base class data structures. Concrete + /// parser classes must override this method. + /// + protected abstract void Initialize(); + + /// + /// Main entry point of the Shift-Reduce Parser. + /// + /// True if parse succeeds, else false for + /// unrecoverable errors + public bool Parse() { + Initialize(); // allow derived classes to instantiate rules, states and nonTerminals + + NextToken = 0; + FsaState = states[0]; + + StateStack.Push( FsaState ); + valueStack.Push( CurrentSemanticValue ); + LocationStack.Push( CurrentLocationSpan ); + + while (true) { +#if TRACE_ACTIONS + Console.Error.WriteLine("Entering state {0} ", FsaState.number); + DisplayStack(); +#endif + int action = FsaState.defaultAction; + + if (FsaState.ParserTable != null) { + if (NextToken == 0) { + // We save the last token span, so that the location span + // of production right hand sides that begin or end with a + // nullable production will be correct. + LastSpan = scanner.yylloc; + NextToken = scanner.yylex(); +#if TRACE_ACTIONS + Console.Error.WriteLine( "Reading: Next token is {0}", TerminalToString( NextToken ) ); +#endif + } +#if TRACE_ACTIONS + else + Console.Error.WriteLine( "Next token is still {0}", TerminalToString( NextToken ) ); +#endif + if (FsaState.ParserTable.ContainsKey( NextToken )) + action = FsaState.ParserTable[NextToken]; + } + + if (action > 0) // shift + { + Shift( action ); + } + else if (action < 0) // reduce + { + try { + Reduce( -action ); + if (action == -1) // accept + return true; + } + catch (AbortException) { + return false; + } + catch (AcceptException) { + return true; + } + catch (ErrorException) { + if (!ErrorRecovery()) + return false; + else + throw; // Rethrow x, preserving information. + } + } + else if (action == 0 && !ErrorRecovery()) // error + return false; + } + } + + private void Shift( int stateIndex ) { +#if TRACE_ACTIONS + Console.Error.Write("Shifting token {0}, ", TerminalToString(NextToken)); +#endif + FsaState = states[stateIndex]; + + valueStack.Push( scanner.yylval ); + StateStack.Push( FsaState ); + LocationStack.Push( scanner.yylloc ); + + if (recovering) { + if (NextToken != errorToken) + tokensSinceLastError++; + + if (tokensSinceLastError > 5) + recovering = false; + } + + if (NextToken != endOfFileToken) + NextToken = 0; + } + + private void Reduce( int ruleNumber ) { +#if TRACE_ACTIONS + DisplayRule(ruleNumber); +#endif + Rule rule = rules[ruleNumber]; + int rhLen = rule.RightHandSide.Length; + // + // Default actions for unit productions. + // + if (rhLen == 1) { + CurrentSemanticValue = valueStack.TopElement(); // Default action: $$ = $1; + CurrentLocationSpan = LocationStack.TopElement(); // Default action "@$ = @1; + } + else if (rhLen == 0) { + // Create a new blank value. + // Explicit semantic action may mutate this value + CurrentSemanticValue = default( TValue ); + // The location span for an empty production will start with the + // beginning of the next lexeme, and end with the finish of the + // previous lexeme. This gives the correct behaviour when this + // nonsense value is used in later Merge operations. + CurrentLocationSpan = (scanner.yylloc != null && LastSpan != null ? + scanner.yylloc.Merge( LastSpan ) : + default( TSpan )); + } + else { + // Default action: $$ = $1; + CurrentSemanticValue = valueStack[LocationStack.Depth - rhLen]; + // Default action "@$ = @1.Merge(@N)" for location info. + TSpan at1 = LocationStack[LocationStack.Depth - rhLen]; + TSpan atN = LocationStack[LocationStack.Depth - 1]; + CurrentLocationSpan = + ((at1 != null && atN != null) ? at1.Merge( atN ) : default( TSpan )); + } + + DoAction( ruleNumber ); + + for (int i = 0; i < rule.RightHandSide.Length; i++) { + StateStack.Pop(); + valueStack.Pop(); + LocationStack.Pop(); + } + FsaState = StateStack.TopElement(); + + if (FsaState.Goto.ContainsKey( rule.LeftHandSide )) + FsaState = states[FsaState.Goto[rule.LeftHandSide]]; + + StateStack.Push( FsaState ); + valueStack.Push( CurrentSemanticValue ); + LocationStack.Push( CurrentLocationSpan ); + } + + /// + /// Execute the selected action from array. + /// Must be overriden in derived classes. + /// + /// Index of the action to perform + protected abstract void DoAction( int actionNumber ); + + private bool ErrorRecovery() { + bool discard; + + if (!recovering) // if not recovering from previous error + ReportError(); + + if (!FindErrorRecoveryState()) + return false; + // + // The interim fix for the "looping in error recovery" + // artifact involved moving the setting of the recovering + // bool until after invalid tokens have been discarded. + // + ShiftErrorToken(); + discard = DiscardInvalidTokens(); + recovering = true; + tokensSinceLastError = 0; + return discard; + } + + private void ReportError() { + StringBuilder errorMsg = new StringBuilder(); + errorMsg.AppendFormat( "Syntax error, unexpected {0}", TerminalToString( NextToken ) ); + + if (FsaState.ParserTable.Count < 7) { + bool first = true; + foreach (int terminal in FsaState.ParserTable.Keys) { + if (first) + errorMsg.Append( ", expecting " ); + else + errorMsg.Append( ", or " ); + + errorMsg.Append( TerminalToString( terminal ) ); + first = false; + } + } + scanner.yyerror( errorMsg.ToString() ); + } + + private void ShiftErrorToken() { + int old_next = NextToken; + NextToken = errorToken; + + Shift( FsaState.ParserTable[NextToken] ); + +#if TRACE_ACTIONS + Console.Error.WriteLine("Entering state {0} ", FsaState.number); +#endif + NextToken = old_next; + } + + private bool FindErrorRecoveryState() { + while (true) // pop states until one found that accepts error token + { + if (FsaState.ParserTable != null && + FsaState.ParserTable.ContainsKey( errorToken ) && + FsaState.ParserTable[errorToken] > 0) // shift + return true; + +#if TRACE_ACTIONS + Console.Error.WriteLine("Error: popping state {0}", StateStack.TopElement().number); +#endif + StateStack.Pop(); + valueStack.Pop(); + LocationStack.Pop(); + +#if TRACE_ACTIONS + DisplayStack(); +#endif + if (StateStack.IsEmpty()) { +#if TRACE_ACTIONS + Console.Error.WriteLine("Aborting: didn't find a state that accepts error token"); +#endif + return false; + } + else + FsaState = StateStack.TopElement(); + } + } + + private bool DiscardInvalidTokens() { + + int action = FsaState.defaultAction; + + if (FsaState.ParserTable != null) { + // Discard tokens until find one that works ... + while (true) { + if (NextToken == 0) { +#if TRACE_ACTIONS + Console.Error.Write("Reading a token: "); +#endif + NextToken = scanner.yylex(); + } +#if TRACE_ACTIONS + Console.Error.WriteLine("Next token is {0}", TerminalToString(NextToken)); +#endif + if (NextToken == endOfFileToken) + return false; + + if (FsaState.ParserTable.ContainsKey( NextToken )) + action = FsaState.ParserTable[NextToken]; + + if (action != 0) + return true; + else { +#if TRACE_ACTIONS + Console.Error.WriteLine("Error: Discarding {0}", TerminalToString(NextToken)); +#endif + NextToken = 0; + } + } + } + else if (recovering && tokensSinceLastError == 0) { + // + // Boolean recovering is not set until after the first + // error token has been shifted. Thus if we get back + // here with recovering set and no tokens read we are + // looping on the same error recovery action. This + // happens if current_state.ParserTable is null because + // the state has an LR(0) reduction, but not all + // lookahead tokens are valid. This only occurs for + // error productions that *end* on "error". + // + // This action discards tokens one at a time until + // the looping stops. Another attack would be to always + // use the LALR(1) table if a production ends on "error" + // +#if TRACE_ACTIONS + Console.Error.WriteLine("Error: panic discard of {0}", TerminalToString(NextToken)); +#endif + if (NextToken == endOfFileToken) + return false; + NextToken = 0; + return true; + } + else + return true; + } + + /// + /// Traditional YACC method. Discards the next input token. + /// + [SuppressMessage( "Microsoft.Naming", "CA1709:IdentifiersShouldBeCasedCorrectly", MessageId = "yyclearin" )] + [SuppressMessage( "Microsoft.Naming", "CA1704:IdentifiersShouldBeSpelledCorrectly", MessageId = "yyclearin" )] + // Reason for FxCop message suppression - + // This is a traditional name for YACC-like functionality + protected void yyclearin() { NextToken = 0; } + + /// + /// Tradional YACC method. Clear the "recovering" flag. + /// + [SuppressMessage( "Microsoft.Naming", "CA1709:IdentifiersShouldBeCasedCorrectly", MessageId = "yyerrok" )] + [SuppressMessage( "Microsoft.Naming", "CA1704:IdentifiersShouldBeSpelledCorrectly", MessageId = "yyerrok" )] + // Reason for FxCop message suppression - + // This is a traditional name for YACC-like functionality + protected void yyerrok() { recovering = false; } + + /// + /// OBSOLETE FOR VERSION 1.4.0 + /// Method used by derived types to insert new + /// state instances in the "states" array. + /// + /// index of the state + /// data for the state + protected void AddState( int stateNumber, State state ) { + states[stateNumber] = state; + state.number = stateNumber; + } + + private void DisplayStack() { + Console.Error.Write( "State stack is now:" ); + for (int i = 0; i < StateStack.Depth; i++) + Console.Error.Write( " {0}", StateStack[i].number ); + Console.Error.WriteLine(); + } + + private void DisplayRule( int ruleNumber ) { + Console.Error.Write( "Reducing stack by rule {0}, ", ruleNumber ); + DisplayProduction( rules[ruleNumber] ); + } + + private void DisplayProduction( Rule rule ) { + if (rule.RightHandSide.Length == 0) + Console.Error.Write( "/* empty */ " ); + else + foreach (int symbol in rule.RightHandSide) + Console.Error.Write( "{0} ", SymbolToString( symbol ) ); + + Console.Error.WriteLine( "-> {0}", SymbolToString( rule.LeftHandSide ) ); + } + + /// + /// Abstract state class naming terminal symbols. + /// This is overridden by derived classes with the + /// name (or alias) to be used in error messages. + /// + /// The terminal ordinal + /// + protected abstract string TerminalToString( int terminal ); + + private string SymbolToString( int symbol ) { + if (symbol < 0) + return nonTerminals[-symbol - 1]; + else + return TerminalToString( symbol ); + } + + /// + /// Return text representation of argument character + /// + /// The character to convert + /// String representation of the character + protected static string CharToString( char input ) { + switch (input) { + case '\a': return @"'\a'"; + case '\b': return @"'\b'"; + case '\f': return @"'\f'"; + case '\n': return @"'\n'"; + case '\r': return @"'\r'"; + case '\t': return @"'\t'"; + case '\v': return @"'\v'"; + case '\0': return @"'\0'"; + default: return string.Format( CultureInfo.InvariantCulture, "'{0}'", input ); + } + } + } + + /// + /// Classes implementing this interface must supply a + /// method that merges two location objects to return + /// a new object of the same type. + /// GPPG-generated parsers have the default location + /// action equivalent to "@$ = @1.Merge(@N);" where N + /// is the right-hand-side length of the production. + /// + /// The Location type +#if EXPORT_GPPG + public interface IMerge +#else + internal interface IMerge +#endif + { + /// + /// Interface method that creates a location object from + /// the current and last object. Typically used to create + /// a location object extending from the start of the @1 + /// object to the end of the @N object. + /// + /// The lexically last object to merge + /// The merged location object + TSpan Merge( TSpan last ); + } + + /// + /// This is the default class that carries location + /// information from the scanner to the parser. + /// If you don't declare "%YYLTYPE Foo" the parser + /// will expect to deal with this type. + /// +#if EXPORT_GPPG + public class LexLocation : IMerge +#else + [SuppressMessage( "Microsoft.Performance", "CA1812:AvoidUninstantiatedInternalClasses" )] + internal class LexLocation : IMerge +#endif + { + private int startLine; // start line + private int startColumn; // start column + private int endLine; // end line + private int endColumn; // end column + + /// + /// The line at which the text span starts. + /// + public int StartLine { get { return startLine; } } + + /// + /// The column at which the text span starts. + /// + public int StartColumn { get { return startColumn; } } + + /// + /// The line on which the text span ends. + /// + public int EndLine { get { return endLine; } } + + /// + /// The column of the first character + /// beyond the end of the text span. + /// + public int EndColumn { get { return endColumn; } } + + /// + /// Default no-arg constructor. + /// + public LexLocation() { } + + /// + /// Constructor for text-span with given start and end. + /// + /// start line + /// start column + /// end line + /// end column + public LexLocation( int sl, int sc, int el, int ec ) { startLine = sl; startColumn = sc; endLine = el; endColumn = ec; } + + /// + /// Create a text location which spans from the + /// start of "this" to the end of the argument "last" + /// + /// The last location in the result span + /// The merged span + public LexLocation Merge( LexLocation last ) { return new LexLocation( this.startLine, this.startColumn, last.endLine, last.endColumn ); } + } + + /// + /// Abstract scanner class that GPPG expects its scanners to + /// extend. + /// + /// Semantic value type YYSTYPE + /// Source location type YYLTYPE +#if EXPORT_GPPG + public abstract class AbstractScanner +#else + internal abstract class AbstractScanner +#endif + where TSpan : IMerge { + /// + /// Lexical value optionally set by the scanner. The value + /// is of the %YYSTYPE type declared in the parser spec. + /// + [SuppressMessage( "Microsoft.Design", "CA1051:DoNotDeclareVisibleInstanceFields" )] + [SuppressMessage( "Microsoft.Naming", "CA1709:IdentifiersShouldBeCasedCorrectly", MessageId = "yylval" )] + [SuppressMessage( "Microsoft.Naming", "CA1704:IdentifiersShouldBeSpelledCorrectly", MessageId = "yylval" )] + // Reason for FxCop message suppression - + // This is a traditional name for YACC-like functionality + // A field must be declared for this value of parametric type, + // since it may be instantiated by a value struct. If it were + // implemented as a property, machine generated code in derived + // types would not be able to select on the returned value. +#pragma warning disable 649 + public TValue yylval; // Lexical value: set by scanner +#pragma warning restore 649 + + /// + /// Current scanner location property. The value is of the + /// type declared by %YYLTYPE in the parser specification. + /// + [SuppressMessage( "Microsoft.Naming", "CA1709:IdentifiersShouldBeCasedCorrectly", MessageId = "yylloc" )] + [SuppressMessage( "Microsoft.Naming", "CA1704:IdentifiersShouldBeSpelledCorrectly", MessageId = "yylloc" )] + // Reason for FxCop message suppression - + // This is a traditional name for YACC-like functionality + public virtual TSpan yylloc { + get { return default( TSpan ); } // Empty implementation allowing + set { /* skip */ } // yylloc to be ignored entirely. + } + + /// + /// Main call point for LEX-like scanners. Returns an int + /// corresponding to the token recognized by the scanner. + /// + /// An int corresponding to the token + [SuppressMessage( "Microsoft.Naming", "CA1709:IdentifiersShouldBeCasedCorrectly", MessageId = "yylex" )] + [SuppressMessage( "Microsoft.Naming", "CA1704:IdentifiersShouldBeSpelledCorrectly", MessageId = "yylex" )] + // Reason for FxCop message suppression - + // This is a traditional name for YACC-like functionality + public abstract int yylex(); + + /// + /// Traditional error reporting provided by LEX-like scanners + /// to their YACC-like clients. + /// + /// Message format string + /// Optional array of args + [SuppressMessage( "Microsoft.Naming", "CA1709:IdentifiersShouldBeCasedCorrectly", MessageId = "yyerror" )] + [SuppressMessage( "Microsoft.Naming", "CA1704:IdentifiersShouldBeSpelledCorrectly", MessageId = "yyerror" )] + // Reason for FxCop message suppression - + // This is a traditional name for YACC-like functionality + public virtual void yyerror( string format, params object[] args ) { } + } + + /// + /// Encapsulated state for the parser. + /// Opaque to users, visible to the tool-generated code. + /// +#if EXPORT_GPPG + public class State + { + /// + /// The number of states in the automaton. + /// + public int number; +#else + internal class State { + /// + /// The index of this state in the states array. + /// + internal int number; +#endif + internal Dictionary ParserTable; // Terminal -> ParseAction + internal Dictionary Goto; // NonTerminal -> State; + internal int defaultAction; // = 0; // ParseAction + + /// + /// State transition data for this state. Pairs of elements of the + /// goto array associate symbol ordinals with next state indices. + /// The actions array is passed to another constructor. + /// + /// The action listc + /// Next state data + public State( int[] actions, int[] goToList ) + : this( actions ) { + Goto = new Dictionary(); + for (int i = 0; i < goToList.Length; i += 2) + Goto.Add( goToList[i], goToList[i + 1] ); + } + + /// + /// Action data for this state. Pairs of elements of the + /// action array associate action ordinals with each of + /// those symbols that have actions in the current state. + /// + /// The action array + public State( int[] actions ) { + ParserTable = new Dictionary(); + for (int i = 0; i < actions.Length; i += 2) + ParserTable.Add( actions[i], actions[i + 1] ); + } + + /// + /// Set the default action for this state. + /// + /// Ordinal of the default action + public State( int defaultAction ) { + this.defaultAction = defaultAction; + } + + /// + /// Set the default action and the state transition table. + /// + /// The default action + /// Transitions from this state + public State( int defaultAction, int[] goToList ) + : this( defaultAction ) { + Goto = new Dictionary(); + for (int i = 0; i < goToList.Length; i += 2) + Goto.Add( goToList[i], goToList[i + 1] ); + } + } + + /// + /// Rule representation at runtime. + /// +#if EXPORT_GPPG + public class Rule +#else + internal class Rule +#endif + { + internal int LeftHandSide; // symbol + internal int[] RightHandSide; // symbols + + /// + /// Rule constructor. This holds the ordinal of + /// the left hand side symbol, and the list of + /// right hand side symbols, in lexical order. + /// + /// The LHS non-terminal + /// The RHS symbols, in lexical order + public Rule( int left, int[] right ) { + this.LeftHandSide = left; + this.RightHandSide = right; + } + } + + /// + /// Stack utility for the shift-reduce parser. + /// GPPG parsers have three instances: + /// (1) The parser state stack, T = QUT.Gppg.State, + /// (2) The semantic value stack, T = TValue, + /// (3) The location stack, T = TSpan. + /// + /// +#if EXPORT_GPPG + public class PushdownPrefixState +#else + internal class PushdownPrefixState +#endif + { + // Note that we cannot use the BCL Stack class + // here as derived types need to index into stacks. + // + private T[] array = new T[8]; + private int tos = 0; + + /// + /// Indexer for values of the stack below the top. + /// + /// index of the element, starting from the bottom + /// the selected element + public T this[int index] { get { return array[index]; } } + + /// + /// The current depth of the stack. + /// + public int Depth { get { return tos; } } + + internal void Push( T value ) { + if (tos >= array.Length) { + T[] newarray = new T[array.Length * 2]; + System.Array.Copy( array, newarray, tos ); + array = newarray; + } + array[tos++] = value; + } + + internal T Pop() { + T rslt = array[--tos]; + array[tos] = default( T ); + return rslt; + } + + internal T TopElement() { return array[tos - 1]; } + + internal bool IsEmpty() { return tos == 0; } + } +} + diff --git a/src/DataStreamSuite/DataStreamExpressions/StatementExpression.cs b/src/DataStreamSuite/DataStreamExpressions/StatementExpression.cs new file mode 100644 index 000000000..211c290bb --- /dev/null +++ b/src/DataStreamSuite/DataStreamExpressions/StatementExpression.cs @@ -0,0 +1,28 @@ +namespace Bts.DataStreamExpressions; + +using BCG; + +public abstract class StatementExpression { + ViewStream? view; + + protected StatementExpression() { } + + public virtual string? print(int startCol, IList activeBindings) => + null; + + public virtual string? printView(int startCol) => + null; + + public ViewStream? getView(IDictionary env) => + view ??= myGetView(env); + + public virtual ViewStream? myGetView(IDictionary env) => + null; + + public static string makeCSString(string s) { + if (s == null) + return "(string) null"; + else + return "\"" + s + "\""; + } +} diff --git a/src/DataStreamSuite/DataStreamExpressions/Usings.cs b/src/DataStreamSuite/DataStreamExpressions/Usings.cs new file mode 100644 index 000000000..4de9547f0 --- /dev/null +++ b/src/DataStreamSuite/DataStreamExpressions/Usings.cs @@ -0,0 +1,4 @@ +global using System; +global using System.Linq; +global using System.Collections.Generic; +global using System.Collections.Immutable; diff --git a/src/DataStreamSuite/DataStreamExpressions/ViewNameExpression.cs b/src/DataStreamSuite/DataStreamExpressions/ViewNameExpression.cs new file mode 100644 index 000000000..5d4e3f6e2 --- /dev/null +++ b/src/DataStreamSuite/DataStreamExpressions/ViewNameExpression.cs @@ -0,0 +1,19 @@ +namespace Bts.DataStreamExpressions; + +using BCG; + +public class ViewNameExpression : StatementExpression { + readonly string id; + + public ViewNameExpression(string id) => + this.id = id; + + public override string print(int startCol, IList activeBindings) => + activeBindings.IndexOf(id) != -1 ? id : "new TableStream(\"" + id + "\", db)"; + + public override string printView(int startCol) => + id; + + public override ViewStream? myGetView(IDictionary env) => + env.TryGetValue(id, out var exp) ? exp.getView(env) : new TableStream(id); +} diff --git a/src/DataStreamSuite/DataStreamQueryAnalyzer/DataStreamQueryAnalyzer.csproj b/src/DataStreamSuite/DataStreamQueryAnalyzer/DataStreamQueryAnalyzer.csproj index 7253701e6..5a92d30c8 100644 --- a/src/DataStreamSuite/DataStreamQueryAnalyzer/DataStreamQueryAnalyzer.csproj +++ b/src/DataStreamSuite/DataStreamQueryAnalyzer/DataStreamQueryAnalyzer.csproj @@ -15,16 +15,8 @@ - - - - - $(Externals)\bin\cstools\Tools.dll - - - - - + + diff --git a/src/DataStreamSuite/DataStreamQueryAnalyzer/MainForm.cs b/src/DataStreamSuite/DataStreamQueryAnalyzer/MainForm.cs index 1b139a6d6..40b760522 100644 --- a/src/DataStreamSuite/DataStreamQueryAnalyzer/MainForm.cs +++ b/src/DataStreamSuite/DataStreamQueryAnalyzer/MainForm.cs @@ -2,6 +2,7 @@ using bts; using bts.utils; using mbts.config; +using mbts.spark; using System; using System.Drawing; @@ -16,7 +17,9 @@ using System.Diagnostics; using BCG.Utilities; // using BCG.Utilities.BlueConfiguration; using BCG.Utilities.General; -using BCG.Evaluator.DataStreamExpression; +using Spark.DataStream; +using Bts.DataStreamExpressions; +// using BCG.Evaluator.DataStreamExpression; // using BCG.Utilities.Win32; namespace BCG @@ -144,10 +147,7 @@ namespace BCG DataStreamExpression oldExp = myExp; try { - BCG.Evaluator.DataStreamExpression.syntax parser = - new BCG.Evaluator.DataStreamExpression.syntax(); - DataStreamExpression exp = (DataStreamExpression) - parser.Parse(expStr); + var exp = DataStreamExpressionsInterpreter.Evaluate(expStr); if (exp == null) throw new Exception("Syntax error. I am afraid I have no further details. Let Pedro know if you are really bothered about it."); @@ -165,7 +165,7 @@ namespace BCG myCrono = new Crono(); myCrono.start(); - (myCurrentStream = myExp.getView(new Hashtable())).onChange += new DataStream.ChangeDelegate(myCallback); + (myCurrentStream = myExp.getView(new Dictionary(0))).onChange += new DataStream.ChangeDelegate(myCallback); myChangeStyle = true; //new SimpleBindableStream(myExp.getView(new Hashtable())).onChange += new DataStream.ChangeDelegate(myCallback); /* @@ -342,7 +342,7 @@ namespace BCG myDataGrid.HorizScrollBarValue = hscrollval; - ArrayList varNames = new ArrayList(); + var varNames = new List(0); string code = myExp.print(9,varNames); myPrintCode(dp.schema ,varNames, code); } @@ -386,7 +386,7 @@ namespace BCG private void MainForm_Closed(object sender, System.EventArgs e) { if (myExp != null) - myExp.getView(new Hashtable()).onChange -= new DataStream.ChangeDelegate(myCallback); + myExp.getView(new Dictionary(0)).onChange -= new DataStream.ChangeDelegate(myCallback); } private void myMenuItemExecuteQueryClick(object sender, System.EventArgs e) diff --git a/src/DataStreamSuite/DataStreamSuite.sln b/src/DataStreamSuite/DataStreamSuite.sln index 5b669cab9..d2f25aa2c 100644 --- a/src/DataStreamSuite/DataStreamSuite.sln +++ b/src/DataStreamSuite/DataStreamSuite.sln @@ -29,6 +29,12 @@ Project("{FAE04EC0-301F-11D3-BF4B-00C04F79EFBC}") = "GuiUtilities", "GuiUtilitie EndProject Project("{FAE04EC0-301F-11D3-BF4B-00C04F79EFBC}") = "ArithmeticExpressions", "ArithmeticExpressions\ArithmeticExpressions.csproj", "{EFE6AD17-2564-43A1-B0E5-A2A8DA173002}" EndProject +Project("{FAE04EC0-301F-11D3-BF4B-00C04F79EFBC}") = "DataStreamExpressions", "DataStreamExpressions\DataStreamExpressions.csproj", "{D46A152D-AD29-4BEF-980B-5896694E312F}" +EndProject +Project("{FAE04EC0-301F-11D3-BF4B-00C04F79EFBC}") = "ColumnMapExpressions", "ColumnMapExpressions\ColumnMapExpressions.csproj", "{A844AB94-195B-4A7F-89CD-9A335985328E}" +EndProject +Project("{FAE04EC0-301F-11D3-BF4B-00C04F79EFBC}") = "ExpressionCore", "ExpressionCore\ExpressionCore.csproj", "{B94AB04B-76D0-4D37-8F2E-7A57A70D289F}" +EndProject Global GlobalSection(SolutionConfigurationPlatforms) = preSolution Debug|Any CPU = Debug|Any CPU @@ -66,10 +72,6 @@ Global {EE53687E-BB62-45C6-A1B0-9B8F16B4E2C1}.Debug|Any CPU.Build.0 = Debug|Any CPU {EE53687E-BB62-45C6-A1B0-9B8F16B4E2C1}.Release|Any CPU.ActiveCfg = Release|Any CPU {EE53687E-BB62-45C6-A1B0-9B8F16B4E2C1}.Release|Any CPU.Build.0 = Release|Any CPU - {60269417-583C-43A1-8AD6-AE82B82A00D3}.Debug|Any CPU.ActiveCfg = Debug|Any CPU - {60269417-583C-43A1-8AD6-AE82B82A00D3}.Debug|Any CPU.Build.0 = Debug|Any CPU - {60269417-583C-43A1-8AD6-AE82B82A00D3}.Release|Any CPU.ActiveCfg = Release|Any CPU - {60269417-583C-43A1-8AD6-AE82B82A00D3}.Release|Any CPU.Build.0 = Release|Any CPU {88E70136-91AE-4F56-B243-8F4642B46D6A}.Debug|Any CPU.ActiveCfg = Debug|x64 {88E70136-91AE-4F56-B243-8F4642B46D6A}.Debug|Any CPU.Build.0 = Debug|x64 {88E70136-91AE-4F56-B243-8F4642B46D6A}.Release|Any CPU.ActiveCfg = Release|x64 @@ -90,5 +92,21 @@ Global {CADBA1B3-0837-44DD-85BB-70351AD3A028}.Debug|Any CPU.Build.0 = Debug|Any CPU {CADBA1B3-0837-44DD-85BB-70351AD3A028}.Release|Any CPU.ActiveCfg = Release|Any CPU {CADBA1B3-0837-44DD-85BB-70351AD3A028}.Release|Any CPU.Build.0 = Release|Any CPU + {EFE6AD17-2564-43A1-B0E5-A2A8DA173002}.Debug|Any CPU.ActiveCfg = Debug|Any CPU + {EFE6AD17-2564-43A1-B0E5-A2A8DA173002}.Debug|Any CPU.Build.0 = Debug|Any CPU + {EFE6AD17-2564-43A1-B0E5-A2A8DA173002}.Release|Any CPU.ActiveCfg = Release|Any CPU + {EFE6AD17-2564-43A1-B0E5-A2A8DA173002}.Release|Any CPU.Build.0 = Release|Any CPU + {D46A152D-AD29-4BEF-980B-5896694E312F}.Debug|Any CPU.ActiveCfg = Debug|Any CPU + {D46A152D-AD29-4BEF-980B-5896694E312F}.Debug|Any CPU.Build.0 = Debug|Any CPU + {D46A152D-AD29-4BEF-980B-5896694E312F}.Release|Any CPU.ActiveCfg = Release|Any CPU + {D46A152D-AD29-4BEF-980B-5896694E312F}.Release|Any CPU.Build.0 = Release|Any CPU + {A844AB94-195B-4A7F-89CD-9A335985328E}.Debug|Any CPU.ActiveCfg = Debug|Any CPU + {A844AB94-195B-4A7F-89CD-9A335985328E}.Debug|Any CPU.Build.0 = Debug|Any CPU + {A844AB94-195B-4A7F-89CD-9A335985328E}.Release|Any CPU.ActiveCfg = Release|Any CPU + {A844AB94-195B-4A7F-89CD-9A335985328E}.Release|Any CPU.Build.0 = Release|Any CPU + {B94AB04B-76D0-4D37-8F2E-7A57A70D289F}.Debug|Any CPU.ActiveCfg = Debug|Any CPU + {B94AB04B-76D0-4D37-8F2E-7A57A70D289F}.Debug|Any CPU.Build.0 = Debug|Any CPU + {B94AB04B-76D0-4D37-8F2E-7A57A70D289F}.Release|Any CPU.ActiveCfg = Release|Any CPU + {B94AB04B-76D0-4D37-8F2E-7A57A70D289F}.Release|Any CPU.Build.0 = Release|Any CPU EndGlobalSection EndGlobal diff --git a/src/DataStreamSuite/ExpressionCore/ExpressionCore.csproj b/src/DataStreamSuite/ExpressionCore/ExpressionCore.csproj new file mode 100644 index 000000000..cfadb03dd --- /dev/null +++ b/src/DataStreamSuite/ExpressionCore/ExpressionCore.csproj @@ -0,0 +1,9 @@ + + + + net7.0 + enable + enable + + + diff --git a/src/DataStreamSuite/ExpressionCore/IParser.cs b/src/DataStreamSuite/ExpressionCore/IParser.cs new file mode 100644 index 000000000..e0cb7561a --- /dev/null +++ b/src/DataStreamSuite/ExpressionCore/IParser.cs @@ -0,0 +1,5 @@ +namespace Bts.ExpressionCore; + +public interface IParser { + T Parse(Stream stream); +} diff --git a/src/DataStreamSuite/ExpressionCore/IParserExtensions.cs b/src/DataStreamSuite/ExpressionCore/IParserExtensions.cs new file mode 100644 index 000000000..66fcd5fda --- /dev/null +++ b/src/DataStreamSuite/ExpressionCore/IParserExtensions.cs @@ -0,0 +1,10 @@ +namespace Bts.ExpressionCore; + +using System.Text; + +public static class ParserExtensions { + public static T Parse(this IParser parser, string text) { + using var ms = new MemoryStream(Encoding.Default.GetBytes(text)); + return parser.Parse(ms); + } +} diff --git a/src/Directory.Build.props b/src/Directory.Build.props index 0b031589a..b70f111ea 100644 --- a/src/Directory.Build.props +++ b/src/Directory.Build.props @@ -34,6 +34,8 @@ $(BaseOutputPath)\obj\$(MSBuildProjectName) $(BaseOutputPath)\obj\$(MSBuildProjectName) $(BaseOutputPath)\obj\$(MSBuildProjectName) + $(BaseOutputPath)\obj\$(MSBuildProjectName) + $(BaseOutputPath)\obj\$(MSBuildProjectName) + + + + + $([MSBuild]::MakeRelative(%(Scanner.RootDir)%(Scanner.Directory), $(ScannerOutputPath)))\%(Scanner.FileName).Scanner.g.cs + + + True + True + %(Scanner.Identity) + + + + + + + + + $([MSBuild]::MakeRelative(%(Parser.RootDir)%(Parser.Directory), + $(ParserOutputPath)))\%(Parser.FileName).Parser.g.cs + /gplex /nolines + + + True + True + %(Parser.Identity) + + + + @@ -103,6 +144,20 @@ + + + + + + + + diff --git a/src/client/spark_core/Collections/GenericDictionaryWrapper.cs b/src/client/spark_core/Collections/GenericDictionaryWrapper.cs index d9aba6b80..684de3691 100644 --- a/src/client/spark_core/Collections/GenericDictionaryWrapper.cs +++ b/src/client/spark_core/Collections/GenericDictionaryWrapper.cs @@ -8,6 +8,9 @@ namespace Spark.Core.Collections { using System.Diagnostics.CodeAnalysis; public static class GenericDictionaryWrapper { + public static GenericDictionaryWrapper Create(IDictionary source) => + Create(source, k => k, v => v); + public static GenericDictionaryWrapper Create( IDictionary source, Func? castKey = null,