Token

- Moved to own module

Exceptions (Lexer)

- Moved to own module

Misc

- Fixed up imports for new `tokens` and `exceptions` (lexer) modules

source/tlang/commandline/commands.d

@@ -10,7 +10,8 @@ import jcli;
 import std.stdio;
 import misc.exceptions : TError;
 import std.exception : ErrnoException;
-import compiler.lexer.core : Lexer, Token;
+import compiler.lexer.core : Lexer;
+import compiler.lexer.tokens : Token;
 import compiler.parsing.core : Parser;
 import compiler.typecheck.core : TypeChecker;
 import gogga;

source/tlang/compiler/core.d

@@ -3,6 +3,7 @@ module compiler.core;
 import gogga;
 import std.conv : to;
 import compiler.lexer.core;
+import compiler.lexer.tokens : Token;
 import std.stdio : File;
 import compiler.parsing.core;
 import compiler.symbols.check;

source/tlang/compiler/lexer/core.d

@@ -2,68 +2,9 @@ module compiler.lexer.core;
 
 import std.container.slist;
 import gogga;
-import std.conv : to;
-import std.string : cmp;
 import std.ascii : isDigit;
-import misc.exceptions : TError;
-
-public enum LexerError
-{
-    EXHAUSTED_CHARACTERS,
-    OTHER
-}
-
-public final class LexerException : TError
-{
-    public const Lexer offendingInstance;
-    public const LexerError errType;
-
-    this(Lexer offendingInstance, LexerError errType = LexerError.OTHER, string msg = "")
-    {
-        string positionString = "("~to!(string)(offendingInstance.line)~", "~to!(string)(offendingInstance.column)~")";
-        super("LexerException("~to!(string)(errType)~")"~(msg.length ? ": "~msg : "")~" at "~positionString);
-        this.offendingInstance = offendingInstance;
-        this.errType = errType;
-    }
-
-    this(Lexer offendingInstance, string msg)
-    {
-        this(offendingInstance, LexerError.OTHER, msg);
-    }
-}
-
-/* TODO: Add Token type (which matches column and position too) */
-public final class Token
-{
-    /* The token */
-    private string token;
-
-    /* Line number information */
-    private ulong line, column;
-
-    this(string token, ulong line, ulong column)
-    {
-        this.token = token;
-        this.line = line;
-        this.column = column;
-    }
-
-    override bool opEquals(Object other)
-    {
-        return cmp(token, (cast(Token)other).getToken()) == 0;
-    }
-
-    override string toString()
-    {
-        /* TODO (Column number): Don't adjust here, do it maybe in the lexer itself */
-        return token~" at ("~to!(string)(line)~", "~to!(string)(column-token.length)~")";
-    }
-
-    public string getToken()
-    {
-        return token;
-    }
-}
+import compiler.lexer.exceptions;
+import compiler.lexer.tokens : Token;
 
 public final class Lexer
 {
@@ -80,6 +21,19 @@ public final class Lexer
     private bool stringMode; /* Whether we are in a string "we are here" or not */
     private bool floatMode; /* Whether or not we are building a floating point constant */
 
+
+    // TODO: Move these all to end, I don't like em here
+    public ulong getLine()
+    {
+        return this.line;
+    }
+
+    public ulong getColumn()
+    {
+        return this.column;
+    }
+
+
     /* The tokens */
     private Token[] tokens;
 

source/tlang/compiler/lexer/exceptions.d

@@ -0,0 +1,30 @@
+module compiler.lexer.exceptions;
+
+import misc.exceptions : TError;
+import compiler.lexer.core : Lexer;
+import std.conv : to;
+
+public enum LexerError
+{
+    EXHAUSTED_CHARACTERS,
+    OTHER
+}
+
+public final class LexerException : TError
+{
+    public const Lexer offendingInstance;
+    public const LexerError errType;
+
+    this(Lexer offendingInstance, LexerError errType = LexerError.OTHER, string msg = "")
+    {
+        string positionString = "("~to!(string)(offendingInstance.getLine())~", "~to!(string)(offendingInstance.getColumn())~")";
+        super("LexerException("~to!(string)(errType)~")"~(msg.length ? ": "~msg : "")~" at "~positionString);
+        this.offendingInstance = offendingInstance;
+        this.errType = errType;
+    }
+
+    this(Lexer offendingInstance, string msg)
+    {
+        this(offendingInstance, LexerError.OTHER, msg);
+    }
+}

source/tlang/compiler/lexer/tokens.d

@@ -0,0 +1,36 @@
+module compiler.lexer.tokens;
+
+import std.string : cmp;
+import std.conv : to;
+
+public final class Token
+{
+    /* The token */
+    private string token;
+
+    /* Line number information */
+    private ulong line, column;
+
+    this(string token, ulong line, ulong column)
+    {
+        this.token = token;
+        this.line = line;
+        this.column = column;
+    }
+
+    override bool opEquals(Object other)
+    {
+        return cmp(token, (cast(Token)other).getToken()) == 0;
+    }
+
+    override string toString()
+    {
+        /* TODO (Column number): Don't adjust here, do it maybe in the lexer itself */
+        return token~" at ("~to!(string)(line)~", "~to!(string)(column-token.length)~")";
+    }
+
+    public string getToken()
+    {
+        return token;
+    }
+}

source/tlang/compiler/parsing/core.d

@@ -5,7 +5,7 @@ import std.conv : to;
 import std.string : isNumeric, cmp;
 import compiler.symbols.check;
 import compiler.symbols.data;
-import compiler.lexer.core : Token;
+import compiler.lexer.tokens : Token;
 import core.stdc.stdlib;
 import misc.exceptions : TError;
 import compiler.parsing.exceptions;

source/tlang/compiler/parsing/exceptions.d

@@ -4,7 +4,7 @@ import compiler.parsing.core;
 import misc.exceptions;
 import compiler.symbols.check;
 import compiler.symbols.data;
-import compiler.lexer.core : Token;
+import compiler.lexer.tokens : Token;
 import std.conv : to;
 
 public class ParserException : TError

source/tlang/compiler/symbols/check.d

@@ -1,6 +1,6 @@
 module compiler.symbols.check;
 
-import compiler.lexer.core : Token;
+import compiler.lexer.tokens : Token;
 import std.conv : to;
 import std.string : isNumeric, cmp;
 import misc.utils;