tesses50
/
tlang-runtime-compiler
1
0
Fork 0
Code Issues Pull Requests Packages Projects Releases Wiki Activity
tlang-runtime-compiler/TLang.Lexer/Class1.cs

423 lines
15 KiB
C#
Raw Normal View History

First Commit
2023-07-27 03:31:32 +00:00
using System;
using System.Collections.Generic;
using System.IO;
using System.Text;
namespace TLang.Lexer
{
public class Lex
{
TextReader reader;
private Lex(TextReader reader)
{
this.reader = reader;
}
private LexLineInfo lineInfo = new LexLineInfo();
private LexLineInfo beginLInfo = new LexLineInfo();
private List<LexToken> tokens = new List<LexToken>();
private StringBuilder builder = new StringBuilder();
private void Flush()
{
if(builder.Length > 0)
{
tokens.Add(LexToken.Token(builder.ToString()).WithLineInfo(beginLInfo));
builder.Clear();
}
}
private void Append(int c)
{
if(c == -1) return;
if(builder.Length == 0)
beginLInfo = lineInfo.Clone();
builder.Append((char)c);
}
public static LexContext GetTokens(TextReader reader,string filename="memory.tlang")
{
Lex lex = new Lex(reader);
lex.StartLexing();
return new LexContext(lex.tokens);
}
private void ParseChar()
{
var lineInfo = this.lineInfo.Clone();
//'c'
var (mChar,Escaped,EOF) = ReadChar();
tokens.Add(LexToken.Char(mChar.ToString()).WithLineInfo(lineInfo));
_ = Next;
}
private void ParseString(bool interoplated)
{
var lineInfo = this.lineInfo.Clone();
//$"Demi Lovato is { variable + 5 } years Old"; -> String.Join("Demi Lovato is ",( variable + 5)," years Old");
StringBuilder b = new StringBuilder();
var (mChar,Escaped,EOF) = ReadChar();
while((mChar != '\"' || Escaped) && !EOF)
{
b.Append(mChar);
(mChar,Escaped,EOF) = ReadChar();
}
if(interoplated)
{
VM works
2023-07-30 07:55:10 +00:00
First Commit
2023-07-27 03:31:32 +00:00
int e = 0;
int escapeI = 0;
StringBuilder b2 = new StringBuilder();
for(int i = 0;i< b.Length;i++)
{
if(b[i] == '{' )
{
if((i+1 < b.Length && b[i+1] != '{') || escapeI >= 1)
{
if(b2.Length > 0 && escapeI < 1)
{
VM works
2023-07-30 07:55:10 +00:00
if(e > 0) tokens.Add(LexToken.Token("+").WithLineInfo(lineInfo));
First Commit
2023-07-27 03:31:32 +00:00
tokens.Add(LexToken.String(b2.ToString()).WithLineInfo(lineInfo));
b2.Clear();
e++;
}
escapeI++;
if(escapeI > 1)
{
b2.Append('{');
}
}
else
{
b2.Append('{');
i++;
}
}
else if(b[i] == '}')
{
if(escapeI >= 1)
{
escapeI--;
if(b2.Length > 0 && escapeI == 0)
{
VM works
2023-07-30 07:55:10 +00:00
if(e > 0) tokens.Add(LexToken.Token("+").WithLineInfo(lineInfo));
First Commit
2023-07-27 03:31:32 +00:00
tokens.Add(LexToken.Token("(").WithLineInfo(lineInfo));
tokens.AddRange(Lex.GetTokensFromString(b2.ToString(),"compilerGenerated.tlang").Tokens);
tokens.Add(LexToken.Token(")").WithLineInfo(lineInfo));
VM works
2023-07-30 07:55:10 +00:00
tokens.Add(LexToken.Token(".").WithLineInfo(lineInfo));
tokens.Add(LexToken.Token("toString").WithLineInfo(lineInfo));
tokens.Add(LexToken.Token("(").WithLineInfo(lineInfo));
tokens.Add(LexToken.Token(")").WithLineInfo(lineInfo));
First Commit
2023-07-27 03:31:32 +00:00
b2.Clear();
e++;
}
if(escapeI >= 1)
{
b2.Append("}");
}
}
}
else {
b2.Append(b[i]);
}
}
if(b2.Length > 0)
{
if(escapeI > 0)
{
VM works
2023-07-30 07:55:10 +00:00
if(e > 0) tokens.Add(LexToken.Token("+").WithLineInfo(lineInfo));
First Commit
2023-07-27 03:31:32 +00:00
tokens.Add(LexToken.Token("(").WithLineInfo(lineInfo));
tokens.AddRange(Lex.GetTokensFromString(b2.ToString(),"compilerGenerated.tlang").Tokens);
tokens.Add(LexToken.Token(")").WithLineInfo(lineInfo));
VM works
2023-07-30 07:55:10 +00:00
tokens.Add(LexToken.Token(".").WithLineInfo(lineInfo));
tokens.Add(LexToken.Token("toString").WithLineInfo(lineInfo));
tokens.Add(LexToken.Token("(").WithLineInfo(lineInfo));
tokens.Add(LexToken.Token(")").WithLineInfo(lineInfo));
First Commit
2023-07-27 03:31:32 +00:00
b2.Clear();
e++;
}
else
{
VM works
2023-07-30 07:55:10 +00:00
if(e > 0) tokens.Add(LexToken.Token("+").WithLineInfo(lineInfo));
First Commit
2023-07-27 03:31:32 +00:00
tokens.Add(LexToken.String(b2.ToString()).WithLineInfo(lineInfo));
b2.Clear();
e++;
}
}
VM works
2023-07-30 07:55:10 +00:00
First Commit
2023-07-27 03:31:32 +00:00
}
else
{
tokens.Add(LexToken.String(b.ToString()).WithLineInfo(lineInfo));
}
}
private (char Char, bool Escaped,bool EOF) ReadChar()
{
int next = Next;
if(next == -1) return (' ',false,true);
if(next == '\\')
{
next = Next;
if(next == 'n')
{
return ('\n',true,false);
}
else if(next == 'r')
{
return ('\r',true,false);
}
else if(next == 't')
{
return ('\t',true,false);
}
else if(next == 'x')
{
string hexCode = new string(new char[]{(char)Next,(char)Next});
int val= int.Parse(hexCode,System.Globalization.NumberStyles.HexNumber);
return ((char)val,true,false);
}
else
{
return ((char)next,true,false);
}
}
return ((char)next,false,false);
}
private void StartLexing()
{
int c;
while((c=Next) != -1)
{
var peek = Peek;
var lineInfo = this.lineInfo.Clone();
switch(c)
{
case '/':
if(peek == '/')
{
_ = Next;
//single line comment
Flush();
while(Next != '\n');
}
else if(peek == '*')
{
var next = Next;
while((next = Next) != -1)
{
if(next == '*')
{
if(Peek == '/')
{
_ = Next;
break;
}
}
}
}
else if(peek == '^')
{
var next = Next;
StringBuilder b = new StringBuilder();
while((next = Next) != -1)
{
if(next == '^')
{
if(Peek == '/')
{
_ = Next;
break;
}
}
b.Append((char)next);
}
tokens.Add(LexToken.Documentation(b.ToString()).WithLineInfo(lineInfo));
}
else if(peek == '=')
{
_ = Next;
tokens.Add(LexToken.Token($"{(char)c}=").WithLineInfo(lineInfo));
}
else
{
tokens.Add(LexToken.Token($"{(char)c}").WithLineInfo(lineInfo));
}
break;
case '\"':
Flush();
ParseString(false);
break;
case '\'':
Flush();
ParseChar();
break;
case '$':
Flush();
if(peek == '\"')
{
_ = Next;
ParseString(true);
}
break;
case '^':
{
Flush();
if(peek == c)
{
_ = Next;
peek=Peek;
if(peek == '=')
{
_ = Next;
tokens.Add(LexToken.Token($"{(char)c}{(char)c}=").WithLineInfo(lineInfo));
}
else
{
tokens.Add(LexToken.Token($"{(char)c}{(char)c}").WithLineInfo(lineInfo));
}
}
else if(peek == '=')
{
_ = Next;
tokens.Add(LexToken.Token($"{(char)c}=").WithLineInfo(lineInfo));
}
else {
tokens.Add(LexToken.Token($"{(char)c}").WithLineInfo(lineInfo));
}
}
break;
case '+':
case '-':
case '<':
case '>':
case '|':
case '&':
{
Flush();
if(peek == c)
{
_ = Next;
tokens.Add(LexToken.Token($"{(char)c}{(char)c}").WithLineInfo(lineInfo));
}
else if(peek == '=')
{
_ = Next;
tokens.Add(LexToken.Token($"{(char)c}=").WithLineInfo(lineInfo));
}
else {
tokens.Add(LexToken.Token($"{(char)c}").WithLineInfo(lineInfo));
}
}
break;
case '*':
case '%':
case '!':
{
Flush();
if(peek == '=')
{
_ = Next;
tokens.Add(LexToken.Token($"{(char)c}=").WithLineInfo(lineInfo));
}
else {
tokens.Add(LexToken.Token($"{(char)c}").WithLineInfo(lineInfo));
}
}
break;
case '[':
case ']':
case '(':
case ')':
case ':':
case '{':
case '}':
case ';':
case '?':
case '.':
case ',':
{
Flush();
tokens.Add(LexToken.Token($"{(char)c}").WithLineInfo(lineInfo));
break;
}
case '=':
{
Flush();
if(peek == c)
{
_ = Next;
tokens.Add(LexToken.Token($"{(char)c}{(char)c}").WithLineInfo(lineInfo));
}
else if(peek == '>')
{
_ = Next;
tokens.Add(LexToken.Token($"{(char)c}>").WithLineInfo(lineInfo));
}
else {
tokens.Add(LexToken.Token($"{(char)c}").WithLineInfo(lineInfo));
}
}
break;
case '\t':
case '\n':
case '\r':
case ' ':
Flush();
SetSpacesBetweenTrue();
break;
default:
Append(c);
break;
}
}
Flush();
}
private void SetSpacesBetweenTrue()
{
if(tokens.Count > 0)
{
tokens[tokens.Count-1].SpacesBetweenThisAndNext = true;
}
}
private int Next {
get{
int read= reader.Read();
if(read != -1)
lineInfo.AppendChar((char)read);
return read;
}
}
private int Peek => reader.Peek();
public static LexContext GetTokensFromFile(string str)
{
using(var f = File.OpenText(str))
{
return GetTokens(f,str);
}
}
public static LexContext GetTokensFromString(string str,string filename="memory.tlang")
{
return GetTokens(new StringReader(str));
}
}
}