187 lines
6.8 KiB
C++
187 lines
6.8 KiB
C++
|
|
#include <list>
|
|
#include <regex>
|
|
|
|
|
|
// Generated from Python3Lexer.g4 by ANTLR 4.13.1
|
|
|
|
#pragma once
|
|
|
|
|
|
#include "antlr4-runtime.h"
|
|
|
|
|
|
|
|
|
|
class Python3Lexer : public antlr4::Lexer {
|
|
public:
|
|
enum {
|
|
INDENT = 1, DEDENT = 2, STRING = 3, NUMBER = 4, INTEGER = 5, DEF = 6,
|
|
RETURN = 7, IF = 8, ELIF = 9, ELSE = 10, WHILE = 11, FOR = 12, IN = 13,
|
|
OR = 14, AND = 15, NOT = 16, NONE = 17, TRUE = 18, FALSE = 19, CONTINUE = 20,
|
|
BREAK = 21, NEWLINE = 22, NAME = 23, STRING_LITERAL = 24, BYTES_LITERAL = 25,
|
|
DECIMAL_INTEGER = 26, OCT_INTEGER = 27, HEX_INTEGER = 28, BIN_INTEGER = 29,
|
|
FLOAT_NUMBER = 30, IMAG_NUMBER = 31, DOT = 32, ELLIPSIS = 33, STAR = 34,
|
|
OPEN_PAREN = 35, CLOSE_PAREN = 36, COMMA = 37, COLON = 38, SEMI_COLON = 39,
|
|
POWER = 40, ASSIGN = 41, OPEN_BRACK = 42, CLOSE_BRACK = 43, OR_OP = 44,
|
|
XOR = 45, AND_OP = 46, LEFT_SHIFT = 47, RIGHT_SHIFT = 48, ADD = 49,
|
|
MINUS = 50, DIV = 51, MOD = 52, IDIV = 53, NOT_OP = 54, OPEN_BRACE = 55,
|
|
CLOSE_BRACE = 56, LESS_THAN = 57, GREATER_THAN = 58, EQUALS = 59, GT_EQ = 60,
|
|
LT_EQ = 61, NOT_EQ_1 = 62, NOT_EQ_2 = 63, AT = 64, ARROW = 65, ADD_ASSIGN = 66,
|
|
SUB_ASSIGN = 67, MULT_ASSIGN = 68, AT_ASSIGN = 69, DIV_ASSIGN = 70,
|
|
MOD_ASSIGN = 71, AND_ASSIGN = 72, OR_ASSIGN = 73, XOR_ASSIGN = 74, LEFT_SHIFT_ASSIGN = 75,
|
|
RIGHT_SHIFT_ASSIGN = 76, POWER_ASSIGN = 77, IDIV_ASSIGN = 78, SKIP_ = 79,
|
|
UNKNOWN_CHAR = 80
|
|
};
|
|
|
|
explicit Python3Lexer(antlr4::CharStream *input);
|
|
|
|
~Python3Lexer() override;
|
|
|
|
|
|
// A queue where extra tokens are pushed on (see the NEWLINE lexer rule).
|
|
private: std::list<antlr4::Token*> tokens ;
|
|
// The stack that keeps track of the indentation level.
|
|
private: std::stack<int> indents ;
|
|
// The amount of opened braces, brackets and parenthesis.
|
|
private: int opened = 0;
|
|
// The most recently produced token.
|
|
private: antlr4::Token* lastToken = nullptr;
|
|
|
|
public: void emit(std::unique_ptr<antlr4::Token> t) override {
|
|
token.release();
|
|
token=std::move(t);
|
|
|
|
tokens.push_back(token.get());
|
|
// std::cout<<t->toString()<<std::endl;
|
|
}
|
|
|
|
|
|
public: std::unique_ptr<antlr4::Token> nextToken() override {
|
|
// Check if the end-of-file is ahead and there are still some DEDENTS expected.
|
|
if (_input->LA(1) == EOF && !this->indents.empty()) {
|
|
// Remove any trailing EOF tokens from our buffer.
|
|
for(auto i=tokens.rbegin();i!=tokens.rend();){
|
|
auto tmp=i;
|
|
i++;
|
|
if((*tmp)->getType()==EOF){
|
|
tokens.erase(tmp.base());
|
|
}
|
|
}
|
|
|
|
|
|
// First emit an extra line break that serves as the end of the statement.
|
|
std::unique_ptr<antlr4::Token> tmp=commonToken(Python3Lexer::NEWLINE, "\n");
|
|
this->emit(std::move(tmp));
|
|
|
|
// Now emit as much DEDENT tokens as needed.
|
|
while (!indents.empty()) {
|
|
auto tmp=createDedent();
|
|
this->emit(std::move(tmp));
|
|
indents.pop();
|
|
}
|
|
|
|
// Put the EOF back on the token stream.
|
|
this->emit(commonToken(static_cast<int>(Python3Lexer::EOF), "<EOF>"));
|
|
}
|
|
|
|
std::unique_ptr<antlr4::Token> next = Lexer::nextToken();
|
|
|
|
if (next->getChannel() == antlr4::Token::DEFAULT_CHANNEL) {
|
|
// Keep track of the last token on the default channel.
|
|
this->lastToken = next.get();
|
|
}
|
|
if (tokens.empty()) {
|
|
return std::move(next);
|
|
} else{
|
|
next.release();
|
|
auto tmp=tokens.front();
|
|
tokens.pop_front();
|
|
return std::unique_ptr<antlr4::Token>(tmp);
|
|
}
|
|
|
|
}
|
|
|
|
private: std::unique_ptr<antlr4::Token> createDedent() {
|
|
auto dedent = commonToken(Python3Lexer::DEDENT, "");
|
|
dedent->setLine(this->lastToken->getLine());
|
|
return std::move(dedent);
|
|
}
|
|
|
|
private: std::unique_ptr<antlr4::CommonToken> commonToken(int type,std::string text) {
|
|
int stop = this->getCharIndex() - 1;
|
|
int start = text.empty() ? stop : stop - text.length() + 1;
|
|
return std::move(std::unique_ptr<antlr4::CommonToken>(new antlr4::CommonToken({ this, _input },
|
|
type,
|
|
DEFAULT_TOKEN_CHANNEL, start, stop)));
|
|
}
|
|
|
|
// Calculates the indentation of the provided spaces, taking the
|
|
// following rules into account:
|
|
//
|
|
// "Tabs are replaced (from left to right) by one to eight spaces
|
|
// such that the total number of characters up to and including
|
|
// the replacement is a multiple of eight [...]"
|
|
//
|
|
// -- https://docs.python.org/3.1/reference/lexical_analysis.html#indentation
|
|
static int getIndentationCount(std::string spaces) {
|
|
int count = 0;
|
|
for (char ch : spaces) {
|
|
switch (ch) {
|
|
case '\t':
|
|
count += 8 - (count % 8);
|
|
break;
|
|
default:
|
|
// A normal space char.
|
|
count++;
|
|
}
|
|
}
|
|
|
|
return count;
|
|
}
|
|
|
|
bool atStartOfInput() {
|
|
return Lexer::getCharPositionInLine() == 0 && Lexer::getLine() == 1;
|
|
}
|
|
|
|
|
|
std::string getGrammarFileName() const override;
|
|
|
|
const std::vector<std::string>& getRuleNames() const override;
|
|
|
|
const std::vector<std::string>& getChannelNames() const override;
|
|
|
|
const std::vector<std::string>& getModeNames() const override;
|
|
|
|
const antlr4::dfa::Vocabulary& getVocabulary() const override;
|
|
|
|
antlr4::atn::SerializedATNView getSerializedATN() const override;
|
|
|
|
const antlr4::atn::ATN& getATN() const override;
|
|
|
|
void action(antlr4::RuleContext *context, size_t ruleIndex, size_t actionIndex) override;
|
|
|
|
bool sempred(antlr4::RuleContext *_localctx, size_t ruleIndex, size_t predicateIndex) override;
|
|
|
|
// By default the static state used to implement the lexer is lazily initialized during the first
|
|
// call to the constructor. You can call this function if you wish to initialize the static state
|
|
// ahead of time.
|
|
static void initialize();
|
|
|
|
private:
|
|
|
|
// Individual action functions triggered by action() above.
|
|
void NEWLINEAction(antlr4::RuleContext *context, size_t actionIndex);
|
|
void OPEN_PARENAction(antlr4::RuleContext *context, size_t actionIndex);
|
|
void CLOSE_PARENAction(antlr4::RuleContext *context, size_t actionIndex);
|
|
void OPEN_BRACKAction(antlr4::RuleContext *context, size_t actionIndex);
|
|
void CLOSE_BRACKAction(antlr4::RuleContext *context, size_t actionIndex);
|
|
void OPEN_BRACEAction(antlr4::RuleContext *context, size_t actionIndex);
|
|
void CLOSE_BRACEAction(antlr4::RuleContext *context, size_t actionIndex);
|
|
|
|
// Individual semantic predicate functions triggered by sempred() above.
|
|
bool NEWLINESempred(antlr4::RuleContext *_localctx, size_t predicateIndex);
|
|
|
|
};
|
|
|