using System; using System.Collections.Generic; using System.IO; using System.Text; namespace TLang.Lexer { public class Lex { TextReader reader; private Lex(TextReader reader) { this.reader = reader; } private LexLineInfo lineInfo = new LexLineInfo(); private LexLineInfo beginLInfo = new LexLineInfo(); private List tokens = new List(); private StringBuilder builder = new StringBuilder(); private void Flush() { if(builder.Length > 0) { tokens.Add(LexToken.Token(builder.ToString()).WithLineInfo(beginLInfo)); builder.Clear(); } } private void Append(int c) { if(c == -1) return; if(builder.Length == 0) beginLInfo = lineInfo.Clone(); builder.Append((char)c); } public static LexContext GetTokens(TextReader reader,string filename="memory.tlang") { Lex lex = new Lex(reader); lex.StartLexing(); return new LexContext(lex.tokens); } private void ParseChar() { var lineInfo = this.lineInfo.Clone(); //'c' var (mChar,Escaped,EOF) = ReadChar(); tokens.Add(LexToken.Char(mChar.ToString()).WithLineInfo(lineInfo)); _ = Next; } private void ParseString(bool interoplated) { var lineInfo = this.lineInfo.Clone(); //$"Demi Lovato is { variable + 5 } years Old"; -> String.Join("Demi Lovato is ",( variable + 5)," years Old"); StringBuilder b = new StringBuilder(); var (mChar,Escaped,EOF) = ReadChar(); while((mChar != '\"' || Escaped) && !EOF) { b.Append(mChar); (mChar,Escaped,EOF) = ReadChar(); } if(interoplated) { tokens.Add(LexToken.Token("String").WithLineInfo(lineInfo)); tokens.Add(LexToken.Token(".").WithLineInfo(lineInfo)); tokens.Add(LexToken.Token("Concat").WithLineInfo(lineInfo)); tokens.Add(LexToken.Token("(").WithLineInfo(lineInfo)); int e = 0; int escapeI = 0; StringBuilder b2 = new StringBuilder(); for(int i = 0;i< b.Length;i++) { if(b[i] == '{' ) { if((i+1 < b.Length && b[i+1] != '{') || escapeI >= 1) { if(b2.Length > 0 && escapeI < 1) { if(e > 0) tokens.Add(LexToken.Token(",").WithLineInfo(lineInfo)); tokens.Add(LexToken.String(b2.ToString()).WithLineInfo(lineInfo)); b2.Clear(); e++; } escapeI++; if(escapeI > 1) { b2.Append('{'); } } else { b2.Append('{'); i++; } } else if(b[i] == '}') { if(escapeI >= 1) { escapeI--; if(b2.Length > 0 && escapeI == 0) { if(e > 0) tokens.Add(LexToken.Token(",").WithLineInfo(lineInfo)); tokens.Add(LexToken.Token("(").WithLineInfo(lineInfo)); tokens.AddRange(Lex.GetTokensFromString(b2.ToString(),"compilerGenerated.tlang").Tokens); tokens.Add(LexToken.Token(")").WithLineInfo(lineInfo)); b2.Clear(); e++; } if(escapeI >= 1) { b2.Append("}"); } } } else { b2.Append(b[i]); } } if(b2.Length > 0) { if(escapeI > 0) { if(e > 0) tokens.Add(LexToken.Token(",").WithLineInfo(lineInfo)); tokens.Add(LexToken.Token("(").WithLineInfo(lineInfo)); tokens.AddRange(Lex.GetTokensFromString(b2.ToString(),"compilerGenerated.tlang").Tokens); tokens.Add(LexToken.Token(")").WithLineInfo(lineInfo)); b2.Clear(); e++; } else { if(e > 0) tokens.Add(LexToken.Token(",").WithLineInfo(lineInfo)); tokens.Add(LexToken.String(b2.ToString()).WithLineInfo(lineInfo)); b2.Clear(); e++; } } tokens.Add(LexToken.Token(")").WithLineInfo(lineInfo)); } else { tokens.Add(LexToken.String(b.ToString()).WithLineInfo(lineInfo)); } } private (char Char, bool Escaped,bool EOF) ReadChar() { int next = Next; if(next == -1) return (' ',false,true); if(next == '\\') { next = Next; if(next == 'n') { return ('\n',true,false); } else if(next == 'r') { return ('\r',true,false); } else if(next == 't') { return ('\t',true,false); } else if(next == 'x') { string hexCode = new string(new char[]{(char)Next,(char)Next}); int val= int.Parse(hexCode,System.Globalization.NumberStyles.HexNumber); return ((char)val,true,false); } else { return ((char)next,true,false); } } return ((char)next,false,false); } private void StartLexing() { int c; while((c=Next) != -1) { var peek = Peek; var lineInfo = this.lineInfo.Clone(); switch(c) { case '/': if(peek == '/') { _ = Next; //single line comment Flush(); while(Next != '\n'); } else if(peek == '*') { var next = Next; while((next = Next) != -1) { if(next == '*') { if(Peek == '/') { _ = Next; break; } } } } else if(peek == '^') { var next = Next; StringBuilder b = new StringBuilder(); while((next = Next) != -1) { if(next == '^') { if(Peek == '/') { _ = Next; break; } } b.Append((char)next); } tokens.Add(LexToken.Documentation(b.ToString()).WithLineInfo(lineInfo)); } else if(peek == '=') { _ = Next; tokens.Add(LexToken.Token($"{(char)c}=").WithLineInfo(lineInfo)); } else { tokens.Add(LexToken.Token($"{(char)c}").WithLineInfo(lineInfo)); } break; case '\"': Flush(); ParseString(false); break; case '\'': Flush(); ParseChar(); break; case '$': Flush(); if(peek == '\"') { _ = Next; ParseString(true); } break; case '^': { Flush(); if(peek == c) { _ = Next; peek=Peek; if(peek == '=') { _ = Next; tokens.Add(LexToken.Token($"{(char)c}{(char)c}=").WithLineInfo(lineInfo)); } else { tokens.Add(LexToken.Token($"{(char)c}{(char)c}").WithLineInfo(lineInfo)); } } else if(peek == '=') { _ = Next; tokens.Add(LexToken.Token($"{(char)c}=").WithLineInfo(lineInfo)); } else { tokens.Add(LexToken.Token($"{(char)c}").WithLineInfo(lineInfo)); } } break; case '+': case '-': case '<': case '>': case '|': case '&': { Flush(); if(peek == c) { _ = Next; tokens.Add(LexToken.Token($"{(char)c}{(char)c}").WithLineInfo(lineInfo)); } else if(peek == '=') { _ = Next; tokens.Add(LexToken.Token($"{(char)c}=").WithLineInfo(lineInfo)); } else { tokens.Add(LexToken.Token($"{(char)c}").WithLineInfo(lineInfo)); } } break; case '*': case '%': case '!': { Flush(); if(peek == '=') { _ = Next; tokens.Add(LexToken.Token($"{(char)c}=").WithLineInfo(lineInfo)); } else { tokens.Add(LexToken.Token($"{(char)c}").WithLineInfo(lineInfo)); } } break; case '[': case ']': case '(': case ')': case ':': case '{': case '}': case ';': case '?': case '.': case ',': { Flush(); tokens.Add(LexToken.Token($"{(char)c}").WithLineInfo(lineInfo)); break; } case '=': { Flush(); if(peek == c) { _ = Next; tokens.Add(LexToken.Token($"{(char)c}{(char)c}").WithLineInfo(lineInfo)); } else if(peek == '>') { _ = Next; tokens.Add(LexToken.Token($"{(char)c}>").WithLineInfo(lineInfo)); } else { tokens.Add(LexToken.Token($"{(char)c}").WithLineInfo(lineInfo)); } } break; case '\t': case '\n': case '\r': case ' ': Flush(); SetSpacesBetweenTrue(); break; default: Append(c); break; } } Flush(); } private void SetSpacesBetweenTrue() { if(tokens.Count > 0) { tokens[tokens.Count-1].SpacesBetweenThisAndNext = true; } } private int Next { get{ int read= reader.Read(); if(read != -1) lineInfo.AppendChar((char)read); return read; } } private int Peek => reader.Peek(); public static LexContext GetTokensFromFile(string str) { using(var f = File.OpenText(str)) { return GetTokens(f,str); } } public static LexContext GetTokensFromString(string str,string filename="memory.tlang") { return GetTokens(new StringReader(str)); } } }