402 lines
13 KiB
C#
402 lines
13 KiB
C#
using System;
|
|
using System.IO;
|
|
using System.Text;
|
|
using System.Collections;
|
|
using Tools;
|
|
|
|
public class LexerGenerate : TokensGen // used during LexerGenerate
|
|
{
|
|
public LexerGenerate(ErrorHandler eh):base(eh) {}
|
|
public void Emit(Hashtable actions,string actvars,bool nameSpace,bool showDfa)
|
|
{
|
|
if (showDfa)
|
|
{
|
|
for (int j=0;j<states.Count; j++)
|
|
((Dfa)states[j]).Print(Console.Out);
|
|
}
|
|
m_outFile.WriteLine("//%|"+m_outname);
|
|
m_outFile.WriteLine("public class yy"+m_outname+" : Tokens {");
|
|
m_outFile.WriteLine(" public yy"+m_outname+"(ErrorHandler eh):base(eh) { arr = new byte[] { ");
|
|
m_tokens.EmitDfa(m_outFile);
|
|
foreach(string x in m_tokens.tokens.Keys)
|
|
m_outFile.WriteLine(" new Tfactory(this,\""+x+"\",new TCreator("+x+"_factory));");
|
|
m_outFile.WriteLine("}");
|
|
foreach (string y in m_tokens.tokens.Keys)
|
|
m_outFile.WriteLine("public static object "+y+"_factory(Lexer yyl) { return new "+y+"(yyl);}");
|
|
Console.WriteLine("Actions function");
|
|
m_outFile.WriteLine("public override TOKEN OldAction(Lexer yym,string yytext, int action, ref bool reject) {");
|
|
m_outFile.WriteLine(" switch(action) {");
|
|
m_outFile.WriteLine(" case -1: break;");
|
|
IDictionaryEnumerator pos = actions.GetEnumerator();
|
|
for (int m=0;m<actions.Count;m++)
|
|
{
|
|
pos.MoveNext();
|
|
int act = (int)pos.Key;
|
|
NfaNode e = (NfaNode)pos.Value;
|
|
if (e.m_sTerminal.Length!=0 && e.m_sTerminal[0]=='%') // auto token action
|
|
continue;
|
|
m_outFile.WriteLine(" case {0}: {1}",act,ActionTransform(e.m_sTerminal));
|
|
m_outFile.WriteLine(" break;"); // in case m_sTerminal ends with a // comment (quite likely)
|
|
}
|
|
m_outFile.WriteLine(" }");
|
|
m_outFile.WriteLine(" return null;");
|
|
m_outFile.WriteLine("}}");
|
|
m_outFile.WriteLine("public class "+m_outname+":Lexer {");
|
|
m_outFile.WriteLine("public "+m_outname+"():base(new yy"+m_outname+"(new ErrorHandler(false))) {}");
|
|
m_outFile.WriteLine("public "+m_outname+"(ErrorHandler eh):base(new yy"+m_outname+"(eh)) {}");
|
|
m_outFile.WriteLine("public "+m_outname+"(Tokens tks):base(tks){}");
|
|
m_outFile.WriteLine(actvars);
|
|
m_outFile.WriteLine(" }");
|
|
if (nameSpace)
|
|
m_outFile.WriteLine("}");
|
|
}
|
|
// This routine modifies Actions containing "return new MyToken()" to "return new MyToken(yyl)", etc
|
|
// (version 3.0 addition, for thread safety purposes)
|
|
string ActionTransform(string str)
|
|
{
|
|
string ret = "";
|
|
str=str.Replace("REJECT","reject=true");
|
|
while (str!="")
|
|
{
|
|
int a = str.IndexOf("new");
|
|
if (a<0)
|
|
break;
|
|
NonWhite(str,ref a,str.Length);
|
|
White(str,ref a,str.Length);
|
|
int b = a;
|
|
while (char.IsLetter(str[b])||char.IsDigit(str[b])||str[b]=='_' && b<str.Length)
|
|
b++;
|
|
string cls = str.Substring(a,b-a);
|
|
if (!m_tokens.tokens.Contains(cls)&& cls.CompareTo("TOKEN")!=0)
|
|
{
|
|
ret += str.Substring(0,b);
|
|
str = str.Substring(b);
|
|
}
|
|
else
|
|
{
|
|
a = str.IndexOf("(",b,str.Length-b);
|
|
if (a<0)
|
|
break;
|
|
White(str,ref a,str.Length);
|
|
a++;
|
|
ret += str.Substring(0,a)+"yym";
|
|
if (str[a]!=')')
|
|
ret += ",";
|
|
str = str.Substring(a);
|
|
}
|
|
}
|
|
return FixActions(ret+str);
|
|
}
|
|
public bool m_lexerseen = false;
|
|
CsReader m_inFile; // the input script
|
|
Hashtable m_actions = new Hashtable(); // int -> NfaNode
|
|
Hashtable m_startstates = new Hashtable(); // string -> NfaNode
|
|
string m_actvars = "";
|
|
bool m_namespace = false;
|
|
LineManager m_LineManager = new LineManager();
|
|
public override int line(int pos) { return m_LineManager.line(pos); }
|
|
public override int position(int pos) { return m_LineManager.position(pos); }
|
|
public override string Saypos(int pos) { return m_LineManager.saypos(pos); }
|
|
|
|
void CopyCode()
|
|
{
|
|
for(;;)
|
|
{ // each line
|
|
int c = m_inFile.Read();
|
|
if (c=='%')
|
|
{
|
|
int d = m_inFile.Read();
|
|
if (d=='}')
|
|
return;
|
|
m_outFile.Write((char)c);
|
|
c = d;
|
|
}
|
|
while (c!='\n')
|
|
{
|
|
m_outFile.Write((char)c);
|
|
c = m_inFile.Read();
|
|
}
|
|
m_outFile.WriteLine();
|
|
}
|
|
}
|
|
void GetRegex(string b, ref int p,int max)
|
|
{
|
|
bool brack = false;
|
|
int quote = 0;
|
|
for (;p<max;p++)
|
|
if (b[p]=='\\')
|
|
p++;
|
|
else if (quote==0 && b[p]=='[')
|
|
brack = true;
|
|
else if (brack && b[p]==']')
|
|
brack = false;
|
|
else if (b[p]==quote)
|
|
quote = 0;
|
|
else if (!brack && quote==0 && (b[p]=='\'' || b[p]=='"'))
|
|
quote = b[p];
|
|
else if (!brack && quote==0 && (b[p]==' '||b[p]=='\t'))
|
|
break;
|
|
}
|
|
|
|
string NewConstructor(TokClassDef pT, string str)
|
|
{
|
|
if (str=="")
|
|
return "";
|
|
string bas = "TOKEN";
|
|
if (pT!=null)
|
|
bas = pT.m_name;
|
|
string newname = "";
|
|
char[] buf = new char[32];
|
|
for (int variant=1;;variant++)
|
|
{ // ensure we get a unique name
|
|
newname = String.Format("{0}_{1}",bas,variant);
|
|
object o = m_tokens.tokens[newname];
|
|
if (o==null)
|
|
break;
|
|
}
|
|
pT = new TokClassDef(this,newname,bas);
|
|
m_outFile.WriteLine("[Serializable] public class "+newname+" : "+bas+" {");
|
|
m_outFile.Write(" public "+newname+"(Lexer yym):base(yym) ");
|
|
pT.m_initialisation = FixActions(str);
|
|
m_outFile.WriteLine(str + "}");
|
|
return newname;
|
|
}
|
|
public byte[] CreateFromContents(string contents) // called from CustomCompilerTools.cs
|
|
{
|
|
m_inFile = new CsReader(contents);
|
|
m_outFile = new StringWriter();
|
|
_Create();
|
|
return m_tokens.m_encoding.GetBytes(m_outFile.ToString());
|
|
}
|
|
public void Create(string fname)
|
|
{
|
|
try
|
|
{
|
|
m_inFile = new CsReader(fname,m_scriptEncoding);
|
|
}
|
|
catch (Exception e)
|
|
{
|
|
if (m_tokens!=null)
|
|
{
|
|
m_tokens.erh.Error(new CSToolsFatalException(4,0,0,"",String.Format("could not open {0} for reading [{1}]",fname,e.Message)));
|
|
return;
|
|
}
|
|
throw(e);
|
|
}
|
|
if (fname.EndsWith(".txt"))
|
|
fname = fname.Substring(0,fname.Length-4);
|
|
try
|
|
{
|
|
m_outFile = new StreamWriter(fname+".cs",false);
|
|
}
|
|
catch (Exception e)
|
|
{
|
|
if (m_tokens!=null)
|
|
{
|
|
m_tokens.erh.Error(new CSToolsFatalException(5,0,0,"",String.Format("could not open {0}.cs for writing [{1}]",fname,e.Message)));
|
|
return;
|
|
}
|
|
throw(e);
|
|
}
|
|
_Create();
|
|
m_outFile.Close();
|
|
}
|
|
void _Create()
|
|
{
|
|
m_outFile.WriteLine("using System;using Tools;");
|
|
m_tokens = new Tokens(erh);
|
|
string buf="";
|
|
string str="";
|
|
string name="";
|
|
string startsym;
|
|
Nfa nfa;
|
|
int p,q,max;
|
|
Console.WriteLine("Reading Input File"); Console.Out.Flush();
|
|
while (!m_inFile.Eof())
|
|
{
|
|
buf = m_inFile.ReadLine();
|
|
startsym = "YYINITIAL";
|
|
max = buf.Length;
|
|
p = 0;
|
|
if (!White(buf,ref p,max))
|
|
continue;
|
|
if (buf[p]=='%')
|
|
{ // directive
|
|
// %lexer
|
|
if(buf.Length>=p+6 && "%lexer".Equals(buf.Substring(p,6)))
|
|
{
|
|
m_lexerseen = true;
|
|
p+=6;
|
|
if (!White(buf,ref p, max))
|
|
continue;
|
|
q = p;
|
|
NonWhite(buf,ref p, max);
|
|
if (q!=p)
|
|
m_outname = buf.Substring(q,p-q);
|
|
continue;
|
|
}
|
|
// %encoding
|
|
if (buf.Length>=p+9 && "%encoding".Equals(buf.Substring(p,9)))
|
|
{
|
|
p+=9; White(buf,ref p, max);
|
|
q = p;
|
|
NonWhite(buf,ref p, max);
|
|
m_tokens.InputEncoding = buf.Substring(q,p-q);
|
|
continue;
|
|
}
|
|
// %namespace
|
|
if (buf.Length>=p+10 && "%namespace".Equals(buf.Substring(p,10)))
|
|
{
|
|
p+=10; White(buf,ref p,max);
|
|
q = p;
|
|
NonWhite(buf,ref p,max);
|
|
m_outFile.WriteLine("namespace "+buf.Substring(q,p-q)+" {");
|
|
m_namespace = true;
|
|
continue;
|
|
}
|
|
// %define
|
|
if(buf.Length>=p+7 && "%define".Equals(buf.Substring(p,7)))
|
|
{
|
|
p+=7; White(buf,ref p,max);
|
|
q = p;
|
|
if (!NonWhite(buf,ref p,max))
|
|
{
|
|
erh.Error(new CSToolsException(44,"Bad define"));
|
|
continue;
|
|
}
|
|
name=buf.Substring(q,p-q);
|
|
p++;
|
|
if (White(buf,ref p,max))
|
|
defines[name]=buf.Substring(p,max-p);
|
|
}
|
|
else
|
|
// % token/node
|
|
if (buf.Length>=p+6 && "%token".Equals(buf.Substring(p,6)))
|
|
EmitClassDefin(buf,ref p,max,m_inFile,"TOKEN", out str,out name,true);
|
|
else if (buf.Length>=p+5 && "%node".Equals(buf.Substring(p,5)))
|
|
EmitClassDefin(buf,ref p,max,m_inFile,"NODE",out str,out name,true);
|
|
else if (buf.Length>=p+2 && "%{".Equals(buf.Substring(p,2)))
|
|
CopyCode();
|
|
else if (buf.Length>=p+9 && "%declare{".Equals(buf.Substring(p,9)))
|
|
{
|
|
p += 8;
|
|
m_actvars = ToBraceIfFound(ref buf,ref p,ref max,m_inFile);
|
|
m_actvars = m_actvars.Substring(1,m_actvars.Length-2);
|
|
}
|
|
else
|
|
m_tokens.erh.Error(new CSToolsException(8,"Unknown directive "+buf.Substring(p,max-p)));
|
|
continue;
|
|
}
|
|
else if (buf[p]=='<')
|
|
{ // startstate
|
|
q = p++;
|
|
while (p<max && buf[p]!='>')
|
|
p++;
|
|
if (p++ ==max)
|
|
{
|
|
m_tokens.erh.Error(new CSToolsException(25,"Bad startsymbol"));
|
|
continue;
|
|
}
|
|
startsym = buf.Substring(q+1,p-q-2);
|
|
White(buf, ref p, max);
|
|
}
|
|
q=p; // can't simply look for nonwhite space here because embedded spaces
|
|
GetRegex(buf,ref p,max);
|
|
string trgx = buf.Substring(q,p-q);
|
|
if (m_tokens.toupper)
|
|
trgx = trgx.ToUpper();
|
|
Regex rgx = new Regex(this,q,trgx);
|
|
Nfa nfa1= new Nfa(this,rgx);
|
|
if (!m_startstates.Contains(startsym))
|
|
m_startstates[startsym] = new Nfa(this);
|
|
nfa = (Nfa)m_startstates[startsym];
|
|
nfa.AddEps(nfa1);
|
|
White(buf,ref p,max);
|
|
m_actions[nfa1.m_end.m_state] = nfa1.m_end;
|
|
// handle multiline actions enclosed in {}
|
|
nfa1.m_end.m_sTerminal = ToBraceIfFound(ref buf,ref p, ref max,m_inFile);
|
|
// examine action string
|
|
if (nfa1.m_end.m_sTerminal.Length>0 && nfa1.m_end.m_sTerminal[0] == '%')
|
|
{
|
|
string tokClass,b = nfa1.m_end.m_sTerminal;
|
|
q = 1;
|
|
max = b.Length;
|
|
int n;
|
|
for (n=0;q<max&&b[q]!=' '&&b[q]!='\t'&&b[q]!='\n'&&b[q]!='{'&&b[q]!=':';q++,n++) // extract the class name
|
|
;
|
|
tokClass = b.Substring(1,n); // new-style auto token construction
|
|
object ob = m_tokens.tokens[tokClass];
|
|
TokClassDef t = (TokClassDef)ob;
|
|
bool isNew = (t==null);
|
|
// check for initialisation action following %name
|
|
string init = b.Substring(n+1,b.Length-n-1);
|
|
string bas1 = "TOKEN";
|
|
bool haveInit = false;
|
|
for (int j=0;j<init.Length;j++)
|
|
if (init[j]=='{')
|
|
{
|
|
haveInit = true;
|
|
break;
|
|
}
|
|
else if (init[j]==':')
|
|
{
|
|
bas1 = "";
|
|
for (;init[j]==' '||init[j]=='\r';j++)
|
|
;
|
|
for (;init[j]!=' '&&init[j]!='\t'&&init[j]!='{'&&init[j]!='\n';j++)
|
|
bas1 += init[j];
|
|
break;
|
|
}
|
|
if (isNew && tokClass!="TOKEN")
|
|
{ // this token class has not been declared. Do so now
|
|
bool isNode = (m_tokens.tokens[bas1]!=null);
|
|
t = new TokClassDef(this,tokClass,bas1); // updates TOKEN.tokens
|
|
m_outFile.WriteLine("//%{0}",tokClass);
|
|
m_outFile.Write("[Serializable] public class {0} : {1}",tokClass,bas1);
|
|
m_outFile.WriteLine("{ public override string yyname() { return \""+tokClass+"\";}");
|
|
m_outFile.WriteLine(" public "+tokClass+"(Lexer yyl):base(yyl) {}}");
|
|
}
|
|
if (haveInit)
|
|
{
|
|
init = init.Replace("yylval","yymval"); // yuk: avoid the next line munging yylval
|
|
init = FixActions(init);
|
|
init = init.Replace("yymval","yylval");
|
|
nfa1.m_end.m_sTerminal = "%"+NewConstructor(t,init);
|
|
}
|
|
}
|
|
}
|
|
if (!m_lexerseen)
|
|
m_tokens.erh.Error(new CSToolsException(26,"No %lexer directive detected: possibly incorrect text encoding?"));
|
|
Console.WriteLine("Constructing DFAs"); Console.Out.Flush();
|
|
foreach (string s in m_startstates.Keys)
|
|
{
|
|
Dfa d = new Dfa((Nfa)m_startstates[s]);
|
|
m_tokens.starts[s] = d;
|
|
if (d.m_actions!=null)
|
|
Console.WriteLine("Warning: This lexer script generates an infinite token stream on bad input");
|
|
}
|
|
Console.WriteLine("Output phase"); Console.Out.Flush();
|
|
Emit(m_actions,m_actvars,m_namespace,m_showDfa);
|
|
Console.WriteLine("End of Create"); Console.Out.Flush();
|
|
if (((Dfa)(m_tokens.starts["YYINITIAL"])).m_actions!=null) // repeat the above warning
|
|
Console.WriteLine("Warning: This lexer script generates an infinite token stream on bad input");
|
|
}
|
|
public static void Main(string[] argv)
|
|
{
|
|
int argc = argv.Length;
|
|
LexerGenerate lexer = new LexerGenerate(new ErrorHandler(true));
|
|
int j;
|
|
for (j = 0; argc>0 && argv[j][0]=='-'; j++,argc--)
|
|
switch(argv[j][1]) {
|
|
case 'D': lexer.m_showDfa = true; break;
|
|
case 'C': lexer.ScriptEncoding = argv[j].Substring(2); break;
|
|
// case 'I': lexer.m_iFlag = argv[j].Substring(2); break;
|
|
}
|
|
if (argc==1)
|
|
lexer.Create(argv[j]);
|
|
else
|
|
lexer.Create("test.lexer");
|
|
Console.WriteLine("LexerGenerator completed successfully");
|
|
}
|
|
}
|