词法分析基础

src/character_util.hpp

@@ -0,0 +1,11 @@
+#pragma once
+
+namespace parser {
+
+bool is_lower_case_letter(char c) {return c >= 'a' && c <= 'z';};
+bool is_capital_letter(char c) {return c >= 'A' && c <= 'Z';};
+bool is_number_character(char c) {return c >= '0' && c <= '9';};
+bool is_letter(char c) {return is_lower_case_letter(c) || is_capital_letter(c);}
+bool is_blank(char c) {return c == ' ' || c == '\t' || c == '\r' || c == '\n';}
+
+} //  namespace parser

src/exceptions.hpp

@@ -0,0 +1,23 @@
+#include <stdexcept>
+#include <string>
+#include "string_util.hpp"
+
+#define DEFINE_EXCEPTION(ClassName) class ClassName: std::runtime_error {\
+public: ClassName(const std::string &message) : std::runtime_error(message) {}\
+};
+
+namespace parser {
+
+DEFINE_EXCEPTION(end_of_file);
+
+class compile_error
+    :public std::runtime_error
+{
+public:
+    compile_error(unsigned int line, const std::string &message) noexcept
+    : std::runtime_error(make_string("compile error at line ", line, ": ", message)) {
+
+    }
+};
+
+} //  namespace parser

src/main.cpp

@@ -1,20 +1,13 @@
-#include <algorithm>
-#include <iostream>
-#include <fstream>
+#include <vector>
 #include <string>
-#include <stack>
+#include <fstream>
+#include "parser.h"
+
+using namespace std;
 
-int main(){
-    std::fstream instream("helloworld.fpp");
-    if(!instream.is_open()){
-        return 1; //Can't open file
+int main(int argc, char** argv) {
+    for(int i = 1; i < argc; ++i) {
+        parser::parser(std::ifstream(argv[i])).parse();
     }
-    std::string code;
-    char c;
-    /*while(!instream.eof()){
-        instream >> c;
-        code.push_back(c);
-    }*/
-    //std::cout << code;
     return 0;
 }

src/parser.cpp

@@ -1,23 +1,135 @@
 #include "parser.h"
+#include "string_util.hpp"
+#include "character_util.hpp"
+#include <iostream>
 
-namespace parser{
-    void Parser::parse(){
-        std::string lastPattern;
-        int i = 0;
-        for (int i = 0; i < code.length(); i++){
-            char ch = code[i];
-            if (ch == ' ' || ch == '\r' || ch == '\n'){
-                if(!lastPattern.empty()){
-                    //Throw error of invalid keyword
+using namespace parser;
+
+namespace parser {
+    char parser::file_scanning_state::get_current_character() noexcept {
+        return currentCharacter;
+    }
+
+    char parser::file_scanning_state::get_next_character() {
+        if(in.eof()) {
+            throw end_of_file("Read failed: end of stream.");
+        }
+        currentCharacter = in.get();
+        return currentCharacter;
+    }
+
+    bool parser::file_scanning_state::eof() noexcept {return in.eof();}
+
+    void parser::scan_tokens() {
+        unsigned int line = 1;
+
+        auto &skip_blanks = [&]() -> unsigned int {
+            unsigned int result = 0;
+            char c = fileScanningState.get_current_character();
+            while(is_blank(c)) {
+                ++result;
+                if(c == '\n') {
+                    ++line;
                 }
-                lastPattern = "";
-                return;
+                c = fileScanningState.get_next_character();
+            }
+            return result;
+        };
+
+
+        auto &read_name = [&]() -> std::string {
+            std::string result = "";
+            char c = fileScanningState.get_current_character();
+            if(is_letter(c) || c == '_') {
+                result += c;
+                c = fileScanningState.get_next_character();
             }
-            if ((!lastPattern.empty())&&((std::string)"function").substr(0,lastPattern.length()-1)==lastPattern){
-                lastPattern.push_back(ch);
-            }else{
-                //Throw error of invalid keyword
+            else {
+                throw std::runtime_error("Is not reading a name.");
+            }
+            while(is_letter(c) || c == '_' || is_number_character(c)) {
+                result += c;
+                c = fileScanningState.get_next_character();
+            }
+            return result;
+        };
+
+        try {
+            fileScanningState.get_next_character();
+            while(!fileScanningState.eof()) {
+                skip_blanks();
+                char c = fileScanningState.get_current_character();
+                if(is_letter(c) || c == '_') {
+                    auto name = read_name();
+                    if(name == "func") {
+                        parsed_tokens.push_back(token_type::function);
+                    }
+                    else {
+                        parsed_tokens.push_back(token_type::name);
+                    }
+                    continue;
+                }
+                else {
+                    switch(c) {
+                    case '[':
+                        parsed_tokens.push_back(token_type::open_square_bracket);
+                        break;
+                    case ']':
+                        parsed_tokens.push_back(token_type::close_square_bracket);
+                        break;
+                    case '(':
+                        parsed_tokens.push_back(token_type::open_bracket);
+                        break;
+                    case ')':
+                        parsed_tokens.push_back(token_type::close_bracket);
+                        break;
+                    case '{':
+                        parsed_tokens.push_back(token_type::open_brace);
+                        break;
+                    case '}':
+                        parsed_tokens.push_back(token_type::close_brace);
+                        break;
+                    default:
+                        throw compile_error(line, make_string("Unexpected character \'", c, "\'."));
+                    }
+                }
+                c = fileScanningState.get_next_character();
+            }
+        }
+        catch (std::exception &e) {
+            std::cerr << "[error] " << e.what() << std::endl;
+        } 
+    }
+
+    void parser::parse() {
+        scan_tokens();
+        for(auto &token : parsed_tokens) {
+            switch(token.type) {
+            case token_type::function:
+                std::cout << "function ";
+                break;
+            case token_type::open_bracket:
+                std::cout << "( ";
+                break;
+            case token_type::close_bracket:
+                std::cout << ") ";
+                break;
+            case token_type::open_square_bracket:
+                std::cout << "[ ";
+                break;
+            case token_type::close_square_bracket:
+                std::cout << "] ";
+                break;
+            case token_type::open_brace:
+                std::cout << "{ ";
+                break;
+            case token_type::close_brace:
+                std::cout << "} ";
+                break;
+            case token_type::name:
+                std::cout << "name ";
             }
         }
+        std::cout << std::flush;
     }
 } // namespace parser

src/parser.h

@@ -1,17 +1,54 @@
-#include <stack>
+#pragma once
+#include <iostream>
+#include <vector>
+#include <memory>
 #include <string>
+#include "exceptions.hpp"
 
 namespace parser
 {
-    class Parser{
-        private:
+    class parser {
         protected:
-            std::string code;
-            int readmode;
+        private:
+            std::istream &in;
+
+            class file_scanning_state {
+            private:
+                std::istream &in;
+                char currentCharacter = 0;
+            public:
+                file_scanning_state(std::istream &in) noexcept: in(in) {}
+                char get_current_character() noexcept;
+                char get_next_character();
+                bool eof() noexcept;
+            }fileScanningState;
+
+            enum class token_type {
+                function,
+                open_bracket,
+                close_bracket,
+                open_square_bracket,
+                close_square_bracket,
+                open_brace,
+                close_brace,
+                name
+            };
+
+            struct extra_data {};
+
+            struct token {
+                token(token_type type) : type(type), extra_data(nullptr) {}
+                token(token_type type, const void * const extra_data) : type(type), extra_data(extra_data) {}
+                token_type type;
+                const void * const extra_data;
+            };
+
+            std::vector<token> parsed_tokens;
+
+            void scan_tokens();
         public:
-            Parser(){}
-            Parser(std::string c) : code(c){}
-            virtual ~Parser(){}
-            virtual void parse();
+            parser(std::istream &in)  noexcept: in(in), fileScanningState(in) {}
+            ~parser() {};
+            void parse();
     };
 } // namespace parser

src/string_util.hpp

@@ -0,0 +1,25 @@
+#pragma once
+#include <string>
+#include <sstream>
+
+namespace parser {
+    template <typename ...Args>
+    std::string make_string(Args... arguments) {
+        static std::stringstream buffer;
+        buffer.clear();
+        buffer.str("");
+        append_string_buffer(buffer, arguments...);
+        return buffer.str();
+    }
+    template <typename T, typename ...Args>
+    void append_string_buffer(std::stringstream &buffer, T argument, Args... arguments) {
+        buffer << argument;
+        append_string_buffer(buffer, arguments...);
+    }
+
+    template <typename T>
+    void append_string_buffer(std::stringstream &buffer, T argument) {
+        buffer << argument;
+    }
+} //  namespace parser
+

test/helloworld.fpp

@@ -1,3 +1,3 @@
-func helloworld(){
+func helloworld() [] {
     say "a"
 }