Added scaffolding work for the OQuery Lexer
This commit is contained in:
parent
bc1b5b5e53
commit
1b1d3797a0
4 changed files with 237 additions and 17 deletions
|
|
@ -7,3 +7,45 @@ FOR ([age == 20] OR [year <= 1998]) AND # FOR conditions should support all
|
|||
CROSS test_table_2 MATCH # Possible values: MATCH (default, implicit), LEFT, RIGHT, FULL
|
||||
IF test_table.id == test_table_2.user_id # Debating if this should support only equality, or more complex conditions, maybe even matching FOR conditions
|
||||
ORDER age, year DESC; # Possible values: ASC (default, implicit), DESC
|
||||
|
||||
|
||||
|
||||
|
||||
|
||||
|
||||
|
||||
|
||||
I start by scanning a character: if that character is a letter
|
||||
or an underscore, I go into keyword/identifier mode, and accumulate until I encounter
|
||||
anything that is not a letter, number, underscore or dot, then check to see if
|
||||
it's a keyword, otherwise it's an identifier (since performance is quite
|
||||
critical in this particular context, I am thinking of fast-tracking directly to
|
||||
identifier by setting a bool flag if either: I am starting or ending
|
||||
with an underscore, or I encounter a dot or a number ...I am going to allow
|
||||
for keyword check if the underscore is found inside, because I might want to
|
||||
have that in the future) ...if a dot is on the end, I error.
|
||||
|
||||
If the initial character is a double quote, I go into string literal mode and
|
||||
accumulate until a non-escaped double quote (inclusive). If I find a
|
||||
new line or the end of the query before that, I error.
|
||||
|
||||
If the initial character is a number, I go into number literal mode and accumulate
|
||||
until I find something that is not a number or a dot. If more than
|
||||
one dot is found, I error. (here I could polish a bit, removing leading zeros if no
|
||||
dot is present and adding a trailing zero if the dot is the last
|
||||
character ...again, is this the job of the lexer? ...maybe not, but maybe it is worth
|
||||
sending cleaner data to the parser, if the fix is this simple
|
||||
....I'll wait to hear what you think)
|
||||
|
||||
If the initial character is any of: =, <, >, !, I check the next character and do
|
||||
the operator-logic you described, and if that doesn't apply then I'll
|
||||
put the single character as a symbol (not used now, but more robust for the future...
|
||||
though I wonder if I should have the parser error on it or error
|
||||
directly in the lexer instead of putting it into a symbol)
|
||||
|
||||
If the initial character is any of the current symbol list: ( ) [ ] { } , * ; I put
|
||||
it as a symbol
|
||||
|
||||
If the initial character is anything else, I error
|
||||
|
||||
when I reach the end, I emit an EndOfInput token
|
||||
|
|
@ -1,17 +0,0 @@
|
|||
enum class eTokenType {
|
||||
Keyword, // GET, IN, FOR, CROSS, IF, ORDER, UNIQUE, AS, OR, AND, NOT,
|
||||
// IN, MATCHES, BETWEEN, INCL, EXCL, INCL_L, INCL_R,
|
||||
// MATCH, LEFT, RIGHT, FULL, ASC, DESC, NULL
|
||||
Identifier, // name, age, test_table.id
|
||||
StringLit, // "Alberto"
|
||||
NumberLit, // 20, 1998
|
||||
Operator, // == != < <= > >=
|
||||
Symbol, // ( ) [ ] { } , . * ;
|
||||
EndOfInput
|
||||
};
|
||||
|
||||
struct Token {
|
||||
eTokenType type;
|
||||
String value; // raw text
|
||||
u32 line, col; // for error reporting — add from day one, you'll regret skipping it
|
||||
};
|
||||
|
|
@ -24,5 +24,129 @@ namespace ostd
|
|||
{
|
||||
namespace db
|
||||
{
|
||||
stdvec<Lexer::Token> Lexer::run(const ostd::String& rawQuery) const
|
||||
{
|
||||
auto l_isLetter = [](char c) -> bool { return (c >= 'A' && c <= 'Z') || (c >= 'a' && c <= 'z'); };
|
||||
auto l_isNumber = [](char c) -> bool { return (c >= '0' && c <= '9'); };
|
||||
auto l_isOperatorStart = [](char c) -> bool { return c == '=' || c == '<' || c == '>' || c == '!'; };
|
||||
auto l_isSymbol = [](char c) -> bool { return c == '(' || c == ')' || c == '[' || c == ']' || c == '{' || c == '}' || c == ',' || c == '*' || c == ';'; };
|
||||
|
||||
set_error(ErrorCodes::NoError);
|
||||
stdvec<Token> tokens;
|
||||
tokens.reserve(InitialTokenCount);
|
||||
|
||||
u32 line = 1, col = 1;
|
||||
for (u32 index = 0; index < rawQuery.len(); index++)
|
||||
{
|
||||
char current = rawQuery[index];
|
||||
Token token;
|
||||
if (current == '\r') continue;
|
||||
else if (current == ' ' || current == '\t')
|
||||
{
|
||||
col++;
|
||||
continue;
|
||||
}
|
||||
else if (current == '\n')
|
||||
{
|
||||
col = 1;
|
||||
line++;
|
||||
continue;
|
||||
}
|
||||
else if (l_isLetter(current) || current == '_')
|
||||
{
|
||||
token = parse_keyword_or_identifier(rawQuery, index, line, col);
|
||||
}
|
||||
else if (current == '\"')
|
||||
{
|
||||
token = parse_string_literal(rawQuery, index, line, col);
|
||||
}
|
||||
else if (l_isNumber(current))
|
||||
{
|
||||
token = parse_number_literal(rawQuery, index, line, col);
|
||||
}
|
||||
else if (l_isOperatorStart(current))
|
||||
{
|
||||
token = parse_operator(rawQuery, index, line, col);
|
||||
}
|
||||
else if (l_isSymbol(current))
|
||||
{
|
||||
token = parse_symbol(rawQuery, index, line, col);
|
||||
}
|
||||
else if (current == '#')
|
||||
{
|
||||
token = parse_comment(rawQuery, index, line, col);
|
||||
}
|
||||
else
|
||||
{
|
||||
token = { eTokenType::Error, "", line, col };
|
||||
set_error(ErrorCodes::InvalidCharacter);
|
||||
}
|
||||
if (token.type == eTokenType::Error)
|
||||
{
|
||||
token.value = get_error_string();
|
||||
return { token };
|
||||
}
|
||||
tokens.push_back(token);
|
||||
}
|
||||
tokens.push_back({ eTokenType::EndOfInput, "", line, col });
|
||||
|
||||
return tokens;
|
||||
}
|
||||
|
||||
String Lexer::get_error_string(void) const
|
||||
{
|
||||
String err_str { "Error " + String::getHexStr(m_error) + ": " };
|
||||
switch (m_error)
|
||||
{
|
||||
case ErrorCodes::InvalidCharacter: err_str += "Invalid character.";
|
||||
case ErrorCodes::NoError:
|
||||
default:
|
||||
err_str = "";
|
||||
}
|
||||
return err_str;
|
||||
}
|
||||
|
||||
Lexer::Token Lexer::parse_keyword_or_identifier(const String& rawQuery, u32& index, u32& line, u32& col) const
|
||||
{
|
||||
Token token { eTokenType::Error, "", line, col };
|
||||
bool identifierOnly { rawQuery[index] == '_' };
|
||||
// TODO
|
||||
return token;
|
||||
}
|
||||
|
||||
Lexer::Token Lexer::parse_string_literal(const String& rawQuery, u32& index, u32& line, u32& col) const
|
||||
{
|
||||
Token token { eTokenType::Error, "", line, col };
|
||||
// TODO
|
||||
return token;
|
||||
}
|
||||
|
||||
Lexer::Token Lexer::parse_number_literal(const String& rawQuery, u32& index, u32& line, u32& col) const
|
||||
{
|
||||
Token token { eTokenType::Error, "", line, col };
|
||||
// TODO
|
||||
return token;
|
||||
}
|
||||
|
||||
Lexer::Token Lexer::parse_operator(const String& rawQuery, u32& index, u32& line, u32& col) const
|
||||
{
|
||||
Token token { eTokenType::Error, "", line, col };
|
||||
// TODO
|
||||
return token;
|
||||
}
|
||||
|
||||
Lexer::Token Lexer::parse_symbol(const String& rawQuery, u32& index, u32& line, u32& col) const
|
||||
{
|
||||
Token token { eTokenType::Error, "", line, col };
|
||||
// TODO
|
||||
return token;
|
||||
}
|
||||
|
||||
Lexer::Token Lexer::parse_comment(const String& rawQuery, u32& index, u32& line, u32& col) const
|
||||
{
|
||||
Token token { eTokenType::Error, "", line, col };
|
||||
// TODO
|
||||
return token;
|
||||
}
|
||||
}
|
||||
}
|
||||
|
|
|
|||
|
|
@ -26,5 +26,76 @@ namespace ostd
|
|||
{
|
||||
namespace db
|
||||
{
|
||||
class Lexer
|
||||
{
|
||||
public: enum class eKeyword { GET, IN, FOR, CROSS, IF, ORDER, UNIQUE, AS, OR, AND, NOT, MATCHES, BETWEEN, INCL, EXCL, INCL_L, INCL_R, MATCH, LEFT, RIGHT, FULL, ASC, DESC };
|
||||
public: enum class eTokenType {
|
||||
Keyword,
|
||||
Identifier,
|
||||
StringLit,
|
||||
NumberLit,
|
||||
Operator, // == != < <= > >=
|
||||
Symbol, // ( ) [ ] { } , . * ;
|
||||
Comment,
|
||||
EndOfInput,
|
||||
|
||||
Error // Used for error reporting
|
||||
};
|
||||
public: struct Token {
|
||||
eTokenType type;
|
||||
String value;
|
||||
u32 line;
|
||||
u32 col;
|
||||
};
|
||||
public: struct ErrorCodes {
|
||||
inline static constexpr u8 NoError { 0x00 };
|
||||
inline static constexpr u8 InvalidCharacter { 0x01 };
|
||||
};
|
||||
|
||||
public:
|
||||
stdvec<Token> run(const String& rawQuery) const;
|
||||
inline bool hasError(void) const { return m_error != ErrorCodes::NoError; }
|
||||
inline u32 getErrorCode(void) const { return m_error; }
|
||||
|
||||
private:
|
||||
inline void set_error(u8 errcode) const { m_error = errcode; }
|
||||
String get_error_string(void) const;
|
||||
|
||||
Token parse_keyword_or_identifier(const String& rawQuery, u32& index, u32& line, u32& col) const;
|
||||
Token parse_string_literal(const String& rawQuery, u32& index, u32& line, u32& col) const;
|
||||
Token parse_number_literal(const String& rawQuery, u32& index, u32& line, u32& col) const;
|
||||
Token parse_operator(const String& rawQuery, u32& index, u32& line, u32& col) const;
|
||||
Token parse_symbol(const String& rawQuery, u32& index, u32& line, u32& col) const;
|
||||
Token parse_comment(const String& rawQuery, u32& index, u32& line, u32& col) const;
|
||||
|
||||
private:
|
||||
mutable u32 m_error { ErrorCodes::NoError };
|
||||
inline static constexpr u32 InitialTokenCount { 200 };
|
||||
inline static const stdumap<String, eKeyword> Keywords {
|
||||
{ "GET", eKeyword::GET },
|
||||
{ "IN", eKeyword::IN },
|
||||
{ "FOR", eKeyword::FOR },
|
||||
{ "CROSS", eKeyword::CROSS },
|
||||
{ "IF", eKeyword::IF },
|
||||
{ "ORDER", eKeyword::ORDER },
|
||||
{ "UNIQUE", eKeyword::UNIQUE },
|
||||
{ "AS", eKeyword::AS },
|
||||
{ "OR", eKeyword::OR },
|
||||
{ "AND", eKeyword::AND },
|
||||
{ "NOT", eKeyword::NOT },
|
||||
{ "MATCHES", eKeyword::MATCHES },
|
||||
{ "BETWEEN", eKeyword::BETWEEN },
|
||||
{ "INCL", eKeyword::INCL },
|
||||
{ "EXCL", eKeyword::EXCL },
|
||||
{ "INCL_L", eKeyword::INCL_L },
|
||||
{ "INCL_R", eKeyword::INCL_R },
|
||||
{ "MATCH", eKeyword::MATCH },
|
||||
{ "LEFT", eKeyword::LEFT },
|
||||
{ "RIGHT", eKeyword::RIGHT },
|
||||
{ "FULL", eKeyword::FULL },
|
||||
{ "ASC", eKeyword::ASC },
|
||||
{ "DESC", eKeyword::DESC },
|
||||
};
|
||||
};
|
||||
}
|
||||
}
|
||||
|
|
|
|||
Loading…
Reference in a new issue