diff --git a/extra/testQuery.oq b/extra/testQuery.oq index 8267b80..d1c9682 100644 --- a/extra/testQuery.oq +++ b/extra/testQuery.oq @@ -7,3 +7,45 @@ FOR ([age == 20] OR [year <= 1998]) AND # FOR conditions should support all CROSS test_table_2 MATCH # Possible values: MATCH (default, implicit), LEFT, RIGHT, FULL IF test_table.id == test_table_2.user_id # Debating if this should support only equality, or more complex conditions, maybe even matching FOR conditions ORDER age, year DESC; # Possible values: ASC (default, implicit), DESC + + + + + + + + +I start by scanning a character: if that character is a letter +or an underscore, I go into keyword/identifier mode, and accumulate until I encounter +anything that is not a letter, number, underscore or dot, then check to see if +it's a keyword, otherwise it's an identifier (since performance is quite +critical in this particular context, I am thinking of fast-tracking directly to +identifier by setting a bool flag if either: I am starting or ending +with an underscore, or I encounter a dot or a number ...I am going to allow +for keyword check if the underscore is found inside, because I might want to +have that in the future) ...if a dot is on the end, I error. + +If the initial character is a double quote, I go into string literal mode and +accumulate until a non-escaped double quote (inclusive). If I find a +new line or the end of the query before that, I error. + +If the initial character is a number, I go into number literal mode and accumulate +until I find something that is not a number or a dot. If more than +one dot is found, I error. (here I could polish a bit, removing leading zeros if no +dot is present and adding a trailing zero if the dot is the last +character ...again, is this the job of the lexer? ...maybe not, but maybe it is worth +sending cleaner data to the parser, if the fix is this simple +....I'll wait to hear what you think) + +If the initial character is any of: =, <, >, !, I check the next character and do +the operator-logic you described, and if that doesn't apply then I'll +put the single character as a symbol (not used now, but more robust for the future... +though I wonder if I should have the parser error on it or error +directly in the lexer instead of putting it into a symbol) + +If the initial character is any of the current symbol list: ( ) [ ] { } , * ; I put +it as a symbol + +If the initial character is anything else, I error + +when I reach the end, I emit an EndOfInput token \ No newline at end of file diff --git a/other/test.txt b/other/test.txt deleted file mode 100644 index d22ec03..0000000 --- a/other/test.txt +++ /dev/null @@ -1,17 +0,0 @@ -enum class eTokenType { - Keyword, // GET, IN, FOR, CROSS, IF, ORDER, UNIQUE, AS, OR, AND, NOT, - // IN, MATCHES, BETWEEN, INCL, EXCL, INCL_L, INCL_R, - // MATCH, LEFT, RIGHT, FULL, ASC, DESC, NULL - Identifier, // name, age, test_table.id - StringLit, // "Alberto" - NumberLit, // 20, 1998 - Operator, // == != < <= > >= - Symbol, // ( ) [ ] { } , . * ; - EndOfInput -}; - -struct Token { - eTokenType type; - String value; // raw text - u32 line, col; // for error reporting — add from day one, you'll regret skipping it -}; diff --git a/src/ostd/io/database/Lexer.cpp b/src/ostd/io/database/Lexer.cpp index 2d34d17..f8ccb80 100644 --- a/src/ostd/io/database/Lexer.cpp +++ b/src/ostd/io/database/Lexer.cpp @@ -24,5 +24,129 @@ namespace ostd { namespace db { + stdvec Lexer::run(const ostd::String& rawQuery) const + { + auto l_isLetter = [](char c) -> bool { return (c >= 'A' && c <= 'Z') || (c >= 'a' && c <= 'z'); }; + auto l_isNumber = [](char c) -> bool { return (c >= '0' && c <= '9'); }; + auto l_isOperatorStart = [](char c) -> bool { return c == '=' || c == '<' || c == '>' || c == '!'; }; + auto l_isSymbol = [](char c) -> bool { return c == '(' || c == ')' || c == '[' || c == ']' || c == '{' || c == '}' || c == ',' || c == '*' || c == ';'; }; + + set_error(ErrorCodes::NoError); + stdvec tokens; + tokens.reserve(InitialTokenCount); + + u32 line = 1, col = 1; + for (u32 index = 0; index < rawQuery.len(); index++) + { + char current = rawQuery[index]; + Token token; + if (current == '\r') continue; + else if (current == ' ' || current == '\t') + { + col++; + continue; + } + else if (current == '\n') + { + col = 1; + line++; + continue; + } + else if (l_isLetter(current) || current == '_') + { + token = parse_keyword_or_identifier(rawQuery, index, line, col); + } + else if (current == '\"') + { + token = parse_string_literal(rawQuery, index, line, col); + } + else if (l_isNumber(current)) + { + token = parse_number_literal(rawQuery, index, line, col); + } + else if (l_isOperatorStart(current)) + { + token = parse_operator(rawQuery, index, line, col); + } + else if (l_isSymbol(current)) + { + token = parse_symbol(rawQuery, index, line, col); + } + else if (current == '#') + { + token = parse_comment(rawQuery, index, line, col); + } + else + { + token = { eTokenType::Error, "", line, col }; + set_error(ErrorCodes::InvalidCharacter); + } + if (token.type == eTokenType::Error) + { + token.value = get_error_string(); + return { token }; + } + tokens.push_back(token); + } + tokens.push_back({ eTokenType::EndOfInput, "", line, col }); + + return tokens; + } + + String Lexer::get_error_string(void) const + { + String err_str { "Error " + String::getHexStr(m_error) + ": " }; + switch (m_error) + { + case ErrorCodes::InvalidCharacter: err_str += "Invalid character."; + case ErrorCodes::NoError: + default: + err_str = ""; + } + return err_str; + } + + Lexer::Token Lexer::parse_keyword_or_identifier(const String& rawQuery, u32& index, u32& line, u32& col) const + { + Token token { eTokenType::Error, "", line, col }; + bool identifierOnly { rawQuery[index] == '_' }; + // TODO + return token; + } + + Lexer::Token Lexer::parse_string_literal(const String& rawQuery, u32& index, u32& line, u32& col) const + { + Token token { eTokenType::Error, "", line, col }; + // TODO + return token; + } + + Lexer::Token Lexer::parse_number_literal(const String& rawQuery, u32& index, u32& line, u32& col) const + { + Token token { eTokenType::Error, "", line, col }; + // TODO + return token; + } + + Lexer::Token Lexer::parse_operator(const String& rawQuery, u32& index, u32& line, u32& col) const + { + Token token { eTokenType::Error, "", line, col }; + // TODO + return token; + } + + Lexer::Token Lexer::parse_symbol(const String& rawQuery, u32& index, u32& line, u32& col) const + { + Token token { eTokenType::Error, "", line, col }; + // TODO + return token; + } + + Lexer::Token Lexer::parse_comment(const String& rawQuery, u32& index, u32& line, u32& col) const + { + Token token { eTokenType::Error, "", line, col }; + // TODO + return token; + } } } diff --git a/src/ostd/io/database/Lexer.hpp b/src/ostd/io/database/Lexer.hpp index c5ed3e4..c399932 100644 --- a/src/ostd/io/database/Lexer.hpp +++ b/src/ostd/io/database/Lexer.hpp @@ -26,5 +26,76 @@ namespace ostd { namespace db { + class Lexer + { + public: enum class eKeyword { GET, IN, FOR, CROSS, IF, ORDER, UNIQUE, AS, OR, AND, NOT, MATCHES, BETWEEN, INCL, EXCL, INCL_L, INCL_R, MATCH, LEFT, RIGHT, FULL, ASC, DESC }; + public: enum class eTokenType { + Keyword, + Identifier, + StringLit, + NumberLit, + Operator, // == != < <= > >= + Symbol, // ( ) [ ] { } , . * ; + Comment, + EndOfInput, + + Error // Used for error reporting + }; + public: struct Token { + eTokenType type; + String value; + u32 line; + u32 col; + }; + public: struct ErrorCodes { + inline static constexpr u8 NoError { 0x00 }; + inline static constexpr u8 InvalidCharacter { 0x01 }; + }; + + public: + stdvec run(const String& rawQuery) const; + inline bool hasError(void) const { return m_error != ErrorCodes::NoError; } + inline u32 getErrorCode(void) const { return m_error; } + + private: + inline void set_error(u8 errcode) const { m_error = errcode; } + String get_error_string(void) const; + + Token parse_keyword_or_identifier(const String& rawQuery, u32& index, u32& line, u32& col) const; + Token parse_string_literal(const String& rawQuery, u32& index, u32& line, u32& col) const; + Token parse_number_literal(const String& rawQuery, u32& index, u32& line, u32& col) const; + Token parse_operator(const String& rawQuery, u32& index, u32& line, u32& col) const; + Token parse_symbol(const String& rawQuery, u32& index, u32& line, u32& col) const; + Token parse_comment(const String& rawQuery, u32& index, u32& line, u32& col) const; + + private: + mutable u32 m_error { ErrorCodes::NoError }; + inline static constexpr u32 InitialTokenCount { 200 }; + inline static const stdumap Keywords { + { "GET", eKeyword::GET }, + { "IN", eKeyword::IN }, + { "FOR", eKeyword::FOR }, + { "CROSS", eKeyword::CROSS }, + { "IF", eKeyword::IF }, + { "ORDER", eKeyword::ORDER }, + { "UNIQUE", eKeyword::UNIQUE }, + { "AS", eKeyword::AS }, + { "OR", eKeyword::OR }, + { "AND", eKeyword::AND }, + { "NOT", eKeyword::NOT }, + { "MATCHES", eKeyword::MATCHES }, + { "BETWEEN", eKeyword::BETWEEN }, + { "INCL", eKeyword::INCL }, + { "EXCL", eKeyword::EXCL }, + { "INCL_L", eKeyword::INCL_L }, + { "INCL_R", eKeyword::INCL_R }, + { "MATCH", eKeyword::MATCH }, + { "LEFT", eKeyword::LEFT }, + { "RIGHT", eKeyword::RIGHT }, + { "FULL", eKeyword::FULL }, + { "ASC", eKeyword::ASC }, + { "DESC", eKeyword::DESC }, + }; + }; } }