423 lines
15 KiB
C#
423 lines
15 KiB
C#
using System;
|
|
using System.Collections.Generic;
|
|
using System.IO;
|
|
using System.Text;
|
|
|
|
namespace TLang.Lexer
|
|
{
|
|
public class Lex
|
|
{
|
|
TextReader reader;
|
|
private Lex(TextReader reader)
|
|
{
|
|
this.reader = reader;
|
|
}
|
|
private LexLineInfo lineInfo = new LexLineInfo();
|
|
private LexLineInfo beginLInfo = new LexLineInfo();
|
|
private List<LexToken> tokens = new List<LexToken>();
|
|
private StringBuilder builder = new StringBuilder();
|
|
private void Flush()
|
|
{
|
|
if(builder.Length > 0)
|
|
{
|
|
tokens.Add(LexToken.Token(builder.ToString()).WithLineInfo(beginLInfo));
|
|
builder.Clear();
|
|
}
|
|
}
|
|
private void Append(int c)
|
|
{
|
|
if(c == -1) return;
|
|
if(builder.Length == 0)
|
|
beginLInfo = lineInfo.Clone();
|
|
|
|
builder.Append((char)c);
|
|
|
|
}
|
|
public static LexContext GetTokens(TextReader reader,string filename="memory.tlang")
|
|
{
|
|
Lex lex = new Lex(reader);
|
|
lex.StartLexing();
|
|
return new LexContext(lex.tokens);
|
|
}
|
|
private void ParseChar()
|
|
{
|
|
var lineInfo = this.lineInfo.Clone();
|
|
//'c'
|
|
var (mChar,Escaped,EOF) = ReadChar();
|
|
|
|
tokens.Add(LexToken.Char(mChar.ToString()).WithLineInfo(lineInfo));
|
|
_ = Next;
|
|
}
|
|
private void ParseString(bool interoplated)
|
|
{
|
|
var lineInfo = this.lineInfo.Clone();
|
|
//$"Demi Lovato is { variable + 5 } years Old"; -> String.Join("Demi Lovato is ",( variable + 5)," years Old");
|
|
|
|
StringBuilder b = new StringBuilder();
|
|
var (mChar,Escaped,EOF) = ReadChar();
|
|
|
|
while((mChar != '\"' || Escaped) && !EOF)
|
|
{
|
|
b.Append(mChar);
|
|
(mChar,Escaped,EOF) = ReadChar();
|
|
}
|
|
if(interoplated)
|
|
{
|
|
|
|
|
|
int e = 0;
|
|
int escapeI = 0;
|
|
|
|
StringBuilder b2 = new StringBuilder();
|
|
for(int i = 0;i< b.Length;i++)
|
|
{
|
|
if(b[i] == '{' )
|
|
{
|
|
if((i+1 < b.Length && b[i+1] != '{') || escapeI >= 1)
|
|
{
|
|
if(b2.Length > 0 && escapeI < 1)
|
|
{
|
|
if(e > 0) tokens.Add(LexToken.Token("+").WithLineInfo(lineInfo));
|
|
tokens.Add(LexToken.String(b2.ToString()).WithLineInfo(lineInfo));
|
|
b2.Clear();
|
|
e++;
|
|
}
|
|
escapeI++;
|
|
if(escapeI > 1)
|
|
{
|
|
b2.Append('{');
|
|
}
|
|
}
|
|
else
|
|
{
|
|
b2.Append('{');
|
|
i++;
|
|
}
|
|
}
|
|
else if(b[i] == '}')
|
|
{
|
|
if(escapeI >= 1)
|
|
{
|
|
escapeI--;
|
|
if(b2.Length > 0 && escapeI == 0)
|
|
{
|
|
if(e > 0) tokens.Add(LexToken.Token("+").WithLineInfo(lineInfo));
|
|
|
|
|
|
tokens.Add(LexToken.Token("(").WithLineInfo(lineInfo));
|
|
tokens.AddRange(Lex.GetTokensFromString(b2.ToString(),"compilerGenerated.tlang").Tokens);
|
|
tokens.Add(LexToken.Token(")").WithLineInfo(lineInfo));
|
|
tokens.Add(LexToken.Token(".").WithLineInfo(lineInfo));
|
|
tokens.Add(LexToken.Token("toString").WithLineInfo(lineInfo));
|
|
tokens.Add(LexToken.Token("(").WithLineInfo(lineInfo));
|
|
tokens.Add(LexToken.Token(")").WithLineInfo(lineInfo));
|
|
|
|
b2.Clear();
|
|
e++;
|
|
}
|
|
if(escapeI >= 1)
|
|
{
|
|
b2.Append("}");
|
|
}
|
|
}
|
|
}
|
|
else {
|
|
b2.Append(b[i]);
|
|
}
|
|
}
|
|
|
|
if(b2.Length > 0)
|
|
{
|
|
if(escapeI > 0)
|
|
{
|
|
if(e > 0) tokens.Add(LexToken.Token("+").WithLineInfo(lineInfo));
|
|
|
|
|
|
tokens.Add(LexToken.Token("(").WithLineInfo(lineInfo));
|
|
tokens.AddRange(Lex.GetTokensFromString(b2.ToString(),"compilerGenerated.tlang").Tokens);
|
|
tokens.Add(LexToken.Token(")").WithLineInfo(lineInfo));
|
|
tokens.Add(LexToken.Token(".").WithLineInfo(lineInfo));
|
|
tokens.Add(LexToken.Token("toString").WithLineInfo(lineInfo));
|
|
tokens.Add(LexToken.Token("(").WithLineInfo(lineInfo));
|
|
tokens.Add(LexToken.Token(")").WithLineInfo(lineInfo));
|
|
|
|
b2.Clear();
|
|
e++;
|
|
}
|
|
else
|
|
{
|
|
if(e > 0) tokens.Add(LexToken.Token("+").WithLineInfo(lineInfo));
|
|
tokens.Add(LexToken.String(b2.ToString()).WithLineInfo(lineInfo));
|
|
b2.Clear();
|
|
e++;
|
|
}
|
|
}
|
|
|
|
|
|
|
|
|
|
}
|
|
else
|
|
{
|
|
tokens.Add(LexToken.String(b.ToString()).WithLineInfo(lineInfo));
|
|
}
|
|
}
|
|
private (char Char, bool Escaped,bool EOF) ReadChar()
|
|
{
|
|
|
|
int next = Next;
|
|
if(next == -1) return (' ',false,true);
|
|
if(next == '\\')
|
|
{
|
|
next = Next;
|
|
if(next == 'n')
|
|
{
|
|
return ('\n',true,false);
|
|
}
|
|
else if(next == 'r')
|
|
{
|
|
return ('\r',true,false);
|
|
}
|
|
else if(next == 't')
|
|
{
|
|
return ('\t',true,false);
|
|
}
|
|
else if(next == 'x')
|
|
{
|
|
string hexCode = new string(new char[]{(char)Next,(char)Next});
|
|
int val= int.Parse(hexCode,System.Globalization.NumberStyles.HexNumber);
|
|
return ((char)val,true,false);
|
|
}
|
|
else
|
|
{
|
|
return ((char)next,true,false);
|
|
}
|
|
}
|
|
return ((char)next,false,false);
|
|
}
|
|
private void StartLexing()
|
|
{
|
|
int c;
|
|
while((c=Next) != -1)
|
|
{
|
|
var peek = Peek;
|
|
var lineInfo = this.lineInfo.Clone();
|
|
switch(c)
|
|
{
|
|
case '/':
|
|
if(peek == '/')
|
|
{
|
|
_ = Next;
|
|
//single line comment
|
|
Flush();
|
|
while(Next != '\n');
|
|
}
|
|
else if(peek == '*')
|
|
{
|
|
var next = Next;
|
|
|
|
while((next = Next) != -1)
|
|
{
|
|
if(next == '*')
|
|
{
|
|
if(Peek == '/')
|
|
{
|
|
_ = Next;
|
|
break;
|
|
}
|
|
}
|
|
}
|
|
}
|
|
else if(peek == '^')
|
|
{
|
|
var next = Next;
|
|
StringBuilder b = new StringBuilder();
|
|
|
|
while((next = Next) != -1)
|
|
{
|
|
if(next == '^')
|
|
{
|
|
if(Peek == '/')
|
|
{
|
|
_ = Next;
|
|
break;
|
|
}
|
|
}
|
|
b.Append((char)next);
|
|
}
|
|
tokens.Add(LexToken.Documentation(b.ToString()).WithLineInfo(lineInfo));
|
|
}
|
|
else if(peek == '=')
|
|
{
|
|
_ = Next;
|
|
tokens.Add(LexToken.Token($"{(char)c}=").WithLineInfo(lineInfo));
|
|
}
|
|
else
|
|
{
|
|
tokens.Add(LexToken.Token($"{(char)c}").WithLineInfo(lineInfo));
|
|
}
|
|
break;
|
|
case '\"':
|
|
Flush();
|
|
ParseString(false);
|
|
break;
|
|
case '\'':
|
|
Flush();
|
|
ParseChar();
|
|
break;
|
|
case '$':
|
|
Flush();
|
|
if(peek == '\"')
|
|
{
|
|
_ = Next;
|
|
ParseString(true);
|
|
}
|
|
break;
|
|
case '^':
|
|
{
|
|
Flush();
|
|
if(peek == c)
|
|
{
|
|
_ = Next;
|
|
peek=Peek;
|
|
if(peek == '=')
|
|
{
|
|
_ = Next;
|
|
tokens.Add(LexToken.Token($"{(char)c}{(char)c}=").WithLineInfo(lineInfo));
|
|
|
|
}
|
|
else
|
|
{
|
|
tokens.Add(LexToken.Token($"{(char)c}{(char)c}").WithLineInfo(lineInfo));
|
|
}
|
|
|
|
}
|
|
else if(peek == '=')
|
|
{
|
|
_ = Next;
|
|
tokens.Add(LexToken.Token($"{(char)c}=").WithLineInfo(lineInfo));
|
|
}
|
|
else {
|
|
tokens.Add(LexToken.Token($"{(char)c}").WithLineInfo(lineInfo));
|
|
}
|
|
}
|
|
break;
|
|
case '+':
|
|
case '-':
|
|
case '<':
|
|
case '>':
|
|
case '|':
|
|
case '&':
|
|
{
|
|
Flush();
|
|
if(peek == c)
|
|
{
|
|
_ = Next;
|
|
tokens.Add(LexToken.Token($"{(char)c}{(char)c}").WithLineInfo(lineInfo));
|
|
}
|
|
else if(peek == '=')
|
|
{
|
|
_ = Next;
|
|
tokens.Add(LexToken.Token($"{(char)c}=").WithLineInfo(lineInfo));
|
|
}
|
|
else {
|
|
tokens.Add(LexToken.Token($"{(char)c}").WithLineInfo(lineInfo));
|
|
}
|
|
}
|
|
break;
|
|
case '*':
|
|
case '%':
|
|
case '!':
|
|
{
|
|
Flush();
|
|
if(peek == '=')
|
|
{
|
|
_ = Next;
|
|
tokens.Add(LexToken.Token($"{(char)c}=").WithLineInfo(lineInfo));
|
|
}
|
|
else {
|
|
tokens.Add(LexToken.Token($"{(char)c}").WithLineInfo(lineInfo));
|
|
}
|
|
}
|
|
break;
|
|
case '[':
|
|
case ']':
|
|
case '(':
|
|
case ')':
|
|
case ':':
|
|
case '{':
|
|
case '}':
|
|
case ';':
|
|
case '?':
|
|
case '.':
|
|
case ',':
|
|
{
|
|
Flush();
|
|
tokens.Add(LexToken.Token($"{(char)c}").WithLineInfo(lineInfo));
|
|
break;
|
|
}
|
|
case '=':
|
|
{
|
|
Flush();
|
|
if(peek == c)
|
|
{
|
|
_ = Next;
|
|
tokens.Add(LexToken.Token($"{(char)c}{(char)c}").WithLineInfo(lineInfo));
|
|
}
|
|
else if(peek == '>')
|
|
{
|
|
_ = Next;
|
|
tokens.Add(LexToken.Token($"{(char)c}>").WithLineInfo(lineInfo));
|
|
}
|
|
else {
|
|
tokens.Add(LexToken.Token($"{(char)c}").WithLineInfo(lineInfo));
|
|
}
|
|
}
|
|
break;
|
|
case '\t':
|
|
case '\n':
|
|
case '\r':
|
|
case ' ':
|
|
Flush();
|
|
SetSpacesBetweenTrue();
|
|
break;
|
|
default:
|
|
Append(c);
|
|
break;
|
|
}
|
|
}
|
|
Flush();
|
|
}
|
|
|
|
private void SetSpacesBetweenTrue()
|
|
{
|
|
if(tokens.Count > 0)
|
|
{
|
|
tokens[tokens.Count-1].SpacesBetweenThisAndNext = true;
|
|
}
|
|
}
|
|
|
|
private int Next {
|
|
get{
|
|
int read= reader.Read();
|
|
if(read != -1)
|
|
lineInfo.AppendChar((char)read);
|
|
return read;
|
|
}
|
|
}
|
|
private int Peek => reader.Peek();
|
|
|
|
public static LexContext GetTokensFromFile(string str)
|
|
{
|
|
using(var f = File.OpenText(str))
|
|
{
|
|
return GetTokens(f,str);
|
|
}
|
|
}
|
|
public static LexContext GetTokensFromString(string str,string filename="memory.tlang")
|
|
{
|
|
return GetTokens(new StringReader(str));
|
|
}
|
|
}
|
|
}
|