Merge pull request #18 from dragonfly-lang/6-implement-scanning-to-fu…

…lfil-test-cases Implemented scanning to fulfil test cases
dragonfly-lang · Jan 5, 2025 · ff31051 · ff31051
2 parents 893ad75 + 068737b
commit ff31051
Show file tree

Hide file tree

Showing 6 changed files with 797 additions and 429 deletions.
diff --git a/CMakeLists.txt b/CMakeLists.txt
@@ -10,7 +10,6 @@ include_directories(include)
 
 set(SOURCES
     src/lexer.cpp
-    src/token.cpp
     src/parser.cpp
     src/codegen.cpp
     src/semantics.cpp

diff --git a/include/dragon/lexer.h b/include/dragon/lexer.h
@@ -1,24 +1,38 @@
-#ifndef LEXER_H
-#define LEXER_H
+#pragma once 
 
+#include <string>
+#include <vector>
+#include <sstream>
+#include <optional>
 #include "token.h"
-#include <stdlib.h>
-#include <string.h>
-#include <stdio.h>
-#include <ctype.h>
-
-typedef struct {
-    char* source;
-    size_t position;
-} Lexer;
-
-Lexer* create_lexer(const char* source);
-Token lex_number(Lexer* lexer);
-Token lex_identifier(Lexer* lexer);
-Token lex_symbol(Lexer* lexer);
-Token lex_string(Lexer* lexer);
-Token is_keyword(Token token);
-TokenList* tokenise(const char* source);
-void free_lexer(Lexer* lexer);
-
-#endif // LEXER_H
+
+class Lexer {
+public:
+    Lexer() = default;
+    Lexer(std::string input);
+
+    std::vector<Token> lex(std::string input);  
+    std::vector<Token> lex();
+
+    void reset();
+
+    Token lex_identifier();
+    Token lex_number();
+    Token lex_string();
+    Token lex_symbol();
+    Token lex_single_line_comment();
+    Token lex_multi_line_comment();
+
+    TokenType get_keyword(std::string input);
+
+private:
+    std::vector<Token> tokens;
+    std::string input;
+    size_t index = 0;
+    size_t line = 1;
+    size_t column = 1;
+
+    std::optional<char> peek() const;
+    std::optional<char> peek_next() const;
+    std::optional<char> advance();
+};
diff --git a/include/dragon/token.h b/include/dragon/token.h
@@ -1,62 +1,6 @@
 #pragma once
 
 #include <string>
-#include <format>
-
-enum class TokenType;
-
-class Token {
-public: 
-    TokenType type;
-    std::string value;
-    size_t line;
-    size_t column;
-
-    Token(TokenType type, std::string value, size_t line, size_t column) {
-        this->type = type;
-        this->value = value;
-        this->line = line;
-        this->column = column;
-    }
-
-    Token(TokenType type, std::string value) {
-        this->type = type;
-        this->value = value;
-        this->line = 0;
-        this->column = 0;
-    }
-
-    Token(TokenType type) {
-        this->type = type;
-        this->value = "";
-        this->line = 0;
-        this->column = 0;
-    }
-
-    inline bool operator==(const Token& other) const {
-        return this->type == other.type && this->value == other.value;
-    }
-
-    inline bool operator!=(const Token& other) const {
-        return this->type != other.type || this->value != other.value;
-    }
-
-    inline std::string to_string() {
-        if (this->line == 0 && this->column == 0 && this->value == "") {
-            return std::format("Token({})", this->type);
-        }
-
-        if (this->line == 0 && this->column == 0) {
-            return std::format("Token({}, {})", this->type, this->value);
-        } 
-
-        if (this->value == "") {
-            return std::format("Token({}, {}, {}, {})", this->type, this->line, this->column);
-        }
-
-        return std::format("Token({}, {}, {}, {})", this->type, this->value, this->line, this->column);
-    }
-};
 
 enum class TokenType {
     // Keywords
@@ -108,3 +52,113 @@ enum class TokenType {
     Comment,
     Unknown,
 };
+
+inline std::string token_type_to_string(TokenType type) {
+    switch (type) {
+        case TokenType::Let: return "Let";
+        case TokenType::Mut: return "Mut";
+        case TokenType::If: return "If";
+        case TokenType::Else: return "Else";
+        case TokenType::While: return "While";
+        case TokenType::For: return "For";
+        case TokenType::In: return "In";
+        case TokenType::True: return "True";
+        case TokenType::False: return "False";
+        case TokenType::IntegerLiteral: return "IntegerLiteral";
+        case TokenType::StringLiteral: return "StringLiteral";
+        case TokenType::Identifier: return "Identifier";
+        case TokenType::Plus: return "Plus";
+        case TokenType::Minus: return "Minus";
+        case TokenType::Star: return "Star";
+        case TokenType::Slash: return "Slash";
+        case TokenType::And: return "And";
+        case TokenType::Or: return "Or";
+        case TokenType::Not: return "Not";
+        case TokenType::Equals: return "Equals";
+        case TokenType::NotEquals: return "NotEquals";
+        case TokenType::LessThan: return "LessThan";
+        case TokenType::GreaterThan: return "GreaterThan";
+        case TokenType::LessThanOrEqualTo: return "LessThanOrEqualTo";
+        case TokenType::GreaterThanOrEqualTo: return "GreaterThanOrEqualTo";
+        case TokenType::Assign: return "Assign";
+        case TokenType::LeftParen: return "LeftParen";
+        case TokenType::RightParen: return "RightParen";
+        case TokenType::LeftBrace: return "LeftBrace";
+        case TokenType::RightBrace: return "RightBrace";
+        case TokenType::LeftBracket: return "LeftBracket";
+        case TokenType::RightBracket: return "RightBracket";
+        case TokenType::Comma: return "Comma";
+        case TokenType::Dot: return "Dot";
+        case TokenType::Range: return "Range";
+        case TokenType::Ampersand: return "Ampersand";
+        case TokenType::Pipe: return "Pipe";
+        case TokenType::Caret: return "Caret";
+        case TokenType::Tilde: return "Tilde";
+        case TokenType::Comment: return "Comment";
+        case TokenType::Unknown: return "Unknown";
+        default: return "Unknown";
+    }
+}
+
+class Token {
+public: 
+    TokenType type;
+    std::string value;
+    size_t line;
+    size_t column;
+
+    Token(TokenType type, std::string value, size_t line, size_t column) {
+        this->type = type;
+        this->value = value;
+        this->line = line;
+        this->column = column;
+    }
+
+    Token(TokenType type, std::string value) {
+        this->type = type;
+        this->value = value;
+        this->line = 0;
+        this->column = 0;
+    }
+
+    Token(TokenType type) {
+        this->type = type;
+        this->value = "";
+        this->line = 0;
+        this->column = 0;
+    }
+
+    inline bool operator==(const Token& other) const {
+        return this->type == other.type && this->value == other.value;
+    }
+
+    inline bool operator!=(const Token& other) const {
+        return this->type != other.type || this->value != other.value;
+    }
+
+    inline std::string to_string() {
+                std::stringstream ss;
+        ss << "Token(" << token_type_to_string(this->type);
+        if (!this->value.empty()) {
+            ss << ", " << this->value;
+        }
+        if (this->line != 0 || this->column != 0) {
+            ss << ", " << this->line << ", " << this->column;
+        }
+        ss << ")";
+        return ss.str();
+    }
+};
+
+inline std::string token_vector_to_string(std::vector<Token> tokens) {
+    std::stringstream ss;
+    ss << "[";
+    for (size_t i = 0; i < tokens.size(); i++) {
+        ss << tokens[i].to_string();
+        if (i < tokens.size() - 1) {
+            ss << ", ";
+        }
+    }
+    ss << "]";
+    return ss.str();
+}