source = p_source_code;

_source = source.get_data();

_current = _source;

_start = _source;

line = 1;

column = 1;

length = p_source_code.length();

position = 0;

ERR_FAIL_COND(indent_stack_stack.is_empty());

indent_stack = indent_stack_stack.back()->get();

indent_stack_stack.pop_back();

if (line < cursor_line) {

return false;

}

if (line > cursor_line) {

return true;

}

if (column < cursor_column) {

return false;

}

return true;

if (unlikely(_is_at_end())) {

return '\0';

}

_current++;

column++;

position++;

if (unlikely(_is_at_end())) {

// Add extra newline even if it's not there, to satisfy the parser.

newline(true);

// Also add needed unindent.

check_indent();

}

return _peek(-1);

if (paren_stack.is_empty()) {

return false;

}

char32_t actual = paren_stack.back()->get();

paren_stack.pop_back();

return actual == p_expected;

Token error = error_stack.back()->get();

error_stack.pop_back();

return error;

Token token(p_type);

token.start_line = start_line;

token.end_line = line;

token.start_column = start_column;

token.end_column = column;

token.source = String::utf32(Span(_start, _current - _start));

if (p_type != Token::ERROR && cursor_line > -1) {

// Also count whitespace after token.

int offset = 0;

while (_peek(offset) == ' ' || _peek(offset) == '\t') {

offset++;

}

int last_column = column + offset;

// Check cursor position in token.

if (start_line == line) {

// Single line token.

if (cursor_line == start_line && cursor_column >= start_column && cursor_column <= last_column) {

if (cursor_column == start_column) {

token.cursor_place = CURSOR_BEGINNING;

} else if (cursor_column < column) {

token.cursor_place = CURSOR_MIDDLE;

} else {

token.cursor_place = CURSOR_END;

}

}

} else {

// Multi line token.

if (cursor_line == start_line && cursor_column >= start_column) {

// Is in first line.

if (cursor_column == start_column) {

token.cursor_place = CURSOR_BEGINNING;

} else {

token.cursor_place = CURSOR_MIDDLE;

}

} else if (cursor_line == line && cursor_column <= last_column) {

// Is in last line.

if (cursor_column < column) {

token.cursor_place = CURSOR_MIDDLE;

} else {

token.cursor_place = CURSOR_END;

}

} else if (cursor_line > start_line && cursor_line < line) {

// Is in middle line.

token.cursor_place = CURSOR_MIDDLE;

}

}

}

last_token = token;

return token;

Token token = make_token(Token::LITERAL);

token.literal = p_literal;

return token;

Token identifier = make_token(Token::IDENTIFIER);

identifier.literal = p_identifier;

return identifier;

Token error = make_token(Token::ERROR);

error.literal = p_message;

return error;

if (paren_stack.is_empty()) {

return make_error(vformat("Closing \"%c\" doesn't have an opening counterpart.", p_paren));

}

Token error = make_error(vformat("Closing \"%c\" doesn't match the opening \"%c\".", p_paren, paren_stack.back()->get()));

paren_stack.pop_back(); // Remove opening one anyway.

return error;

const char32_t *next = _current + 1;

int chars = 2; // Two already matched.

// Test before consuming characters, since we don't want to consume more than needed.

while (*next == p_test) {

chars++;

next++;

}

if (chars >= 7) {

// It is a VCS conflict marker.

while (chars > 1) {

// Consume all characters (first was already consumed by scan()).

_advance();

chars--;

}

return make_token(Token::VCS_CONFLICT_MARKER);

} else {

// It is only a regular double character token, so we consume the second character.

_advance();

return make_token(p_double_type);

}

if (is_unicode_identifier_start(_peek())) {

_advance(); // Consume start character.

} else {

push_error("Expected annotation identifier after \"@\".");

}

while (is_unicode_identifier_continue(_peek())) {

// Consume all identifier characters.

_advance();

}

Token annotation = make_token(Token::ANNOTATION);

annotation.literal = StringName(annotation.source);

return annotation;

#define KEYWORD_LINE(keyword, token_type) keyword,

#define KEYWORD_GROUP_IGNORE(group)

keyword_list = {

KEYWORDS(KEYWORD_GROUP_IGNORE, KEYWORD_LINE)

};

#undef KEYWORD_LINE

#undef KEYWORD_GROUP_IGNORE

bool only_ascii = _peek(-1) < 128;

// Consume all identifier characters.

while (is_unicode_identifier_continue(_peek())) {

char32_t c = _advance();

only_ascii = only_ascii && c < 128;

}

int len = _current - _start;

if (len == 1 && _peek(-1) == '_') {

// Lone underscore.

Token token = make_token(Token::UNDERSCORE);

token.literal = "_";

return token;

}

String name = String::utf32(Span(_start, len));

if (len < MIN_KEYWORD_LENGTH || len > MAX_KEYWORD_LENGTH) {

// Cannot be a keyword, as the length doesn't match any.

return make_identifier(name);

}

if (!only_ascii) {

// Kept here in case the order with push_error matters.

Token id = make_identifier(name);

#ifdef DEBUG_ENABLED

// Additional checks for identifiers but only in debug and if it's available in TextServer.

if (TS->has_feature(TextServer::FEATURE_UNICODE_SECURITY)) {

int64_t confusable = TS->is_confusable(name, keyword_list);

if (confusable >= 0) {

push_error(vformat(R"(Identifier "%s" is visually similar to the GDScript keyword "%s" and thus not allowed.)", name, keyword_list[confusable]));

}

}

#endif // DEBUG_ENABLED

// Cannot be a keyword, as keywords are ASCII only.

return id;

}

// Define some helper macros for the switch case.

#define KEYWORD_GROUP_CASE(char) \

break; \

case char:

#define KEYWORD(keyword, token_type) \

{ \

const int keyword_length = sizeof(keyword) - 1; \

static_assert(keyword_length <= MAX_KEYWORD_LENGTH, "There's a keyword longer than the defined maximum length"); \

static_assert(keyword_length >= MIN_KEYWORD_LENGTH, "There's a keyword shorter than the defined minimum length"); \

if (keyword_length == len && name == keyword) { \

Token kw = make_token(token_type); \

kw.literal = name; \

return kw; \

} \

}

// Find if it's a keyword.

switch (_start[0]) {

default:

KEYWORDS(KEYWORD_GROUP_CASE, KEYWORD)

break;

}

// Check if it's a special literal

if (len == 4) {

if (name == "true") {

return make_literal(true);

} else if (name == "null") {

return make_literal(Variant());

}

} else if (len == 5) {

if (name == "false") {

return make_literal(false);

}

}

// Not a keyword, so must be an identifier.

return make_identifier(name);

#undef KEYWORD_GROUP_CASE

#undef KEYWORD

// Don't overwrite previous newline, nor create if we want a line continuation.

if (p_make_token && !pending_newline && !line_continuation) {

Token newline(Token::NEWLINE);

newline.start_line = line;

newline.end_line = line;

newline.start_column = column - 1;

newline.end_column = column;

pending_newline = true;

last_token = newline;

last_newline = newline;

}

// Increment line/column counters.

line++;

column = 1;

int base = 10;

bool has_decimal = false;

bool has_exponent = false;

bool has_error = false;

bool need_digits = false;

bool (*digit_check_func)(char32_t) = is_digit;

// Sign before hexadecimal or binary.

if ((_peek(-1) == '+' || _peek(-1) == '-') && _peek() == '0') {

_advance();

}

if (_peek(-1) == '.') {

has_decimal = true;

} else if (_peek(-1) == '0') {

if (_peek() == 'x' || _peek() == 'X') {

// Hexadecimal.

base = 16;

digit_check_func = is_hex_digit;

need_digits = true;

_advance();

} else if (_peek() == 'b' || _peek() == 'B') {

// Binary.

base = 2;

digit_check_func = is_binary_digit;

need_digits = true;

_advance();

}

}

if (base != 10 && is_underscore(_peek())) { // Disallow `0x_` and `0b_`.

Token error = make_error(vformat(R"(Unexpected underscore after "0%c".)", _peek(-1)));

error.start_column = column;

error.end_column = column + 1;

push_error(error);

has_error = true;

}

bool previous_was_underscore = false; // Allow `_` to be used in a number, for readability.

while (digit_check_func(_peek()) || is_underscore(_peek())) {

if (is_underscore(_peek())) {

if (previous_was_underscore) {

Token error = make_error(R"(Multiple underscores cannot be adjacent in a numeric literal.)");

error.start_column = column;

error.end_column = column + 1;

push_error(error);

}

previous_was_underscore = true;

} else {

need_digits = false;

previous_was_underscore = false;

}

_advance();

}

// It might be a ".." token (instead of decimal point) so we check if it's not.

if (_peek() == '.' && _peek(1) != '.') {

if (base == 10 && !has_decimal) {

has_decimal = true;

} else if (base == 10) {

Token error = make_error("Cannot use a decimal point twice in a number.");

error.start_column = column;

error.end_column = column + 1;

push_error(error);

has_error = true;

} else if (base == 16) {

Token error = make_error("Cannot use a decimal point in a hexadecimal number.");

error.start_column = column;

error.end_column = column + 1;

push_error(error);

has_error = true;

} else {

Token error = make_error("Cannot use a decimal point in a binary number.");

error.start_column = column;

error.end_column = column + 1;

push_error(error);

has_error = true;

}

if (!has_error) {

_advance();

// Consume decimal digits.

if (is_underscore(_peek())) { // Disallow `10._`, but allow `10.`.

Token error = make_error(R"(Unexpected underscore after decimal point.)");

error.start_column = column;

error.end_column = column + 1;

push_error(error);

has_error = true;

}

previous_was_underscore = false;

while (is_digit(_peek()) || is_underscore(_peek())) {

if (is_underscore(_peek())) {

if (previous_was_underscore) {

Token error = make_error(R"(Multiple underscores cannot be adjacent in a numeric literal.)");

error.start_column = column;

error.end_column = column + 1;

push_error(error);

}

previous_was_underscore = true;

} else {

previous_was_underscore = false;

}

_advance();

}

}

}

if (base == 10) {

if (_peek() == 'e' || _peek() == 'E') {

has_exponent = true;

_advance();

if (_peek() == '+' || _peek() == '-') {

// Exponent sign.

_advance();

}

// Consume exponent digits.

if (!is_digit(_peek())) {

Token error = make_error(R"(Expected exponent value after "e".)");

error.start_column = column;

error.end_column = column + 1;

push_error(error);

}

previous_was_underscore = false;

while (is_digit(_peek()) || is_underscore(_peek())) {

if (is_underscore(_peek())) {

if (previous_was_underscore) {

Token error = make_error(R"(Multiple underscores cannot be adjacent in a numeric literal.)");

error.start_column = column;

error.end_column = column + 1;

push_error(error);

}

previous_was_underscore = true;

} else {

previous_was_underscore = false;

}

_advance();

}

}

}

if (need_digits) {

// No digits in hex or bin literal.

Token error = make_error(vformat(R"(Expected %s digit after "0%c".)", (base == 16 ? "hexadecimal" : "binary"), (base == 16 ? 'x' : 'b')));

error.start_column = column;

error.end_column = column + 1;

return error;

}

// Detect extra decimal point.

if (!has_error && has_decimal && _peek() == '.' && _peek(1) != '.') {

Token error = make_error("Cannot use a decimal point twice in a number.");

error.start_column = column;

error.end_column = column + 1;

push_error(error);

has_error = true;

} else if (is_unicode_identifier_start(_peek()) || is_unicode_identifier_continue(_peek())) {

// Letter at the end of the number.

push_error("Invalid numeric notation.");

}

// Create a string with the whole number.

int len = _current - _start;

String number = String::utf32(Span(_start, len)).remove_char('_');

// Convert to the appropriate literal type.

if (base == 16) {

int64_t value = number.hex_to_int();

return make_literal(value);

} else if (base == 2) {

int64_t value = number.bin_to_int();

return make_literal(value);

} else if (has_decimal || has_exponent) {

double value = number.to_float();

return make_literal(value);

} else {

int64_t value = number.to_int();

return make_literal(value);

}

enum StringType {

STRING_REGULAR,

STRING_NAME,

STRING_NODEPATH,

};

bool is_raw = false;

bool is_multiline = false;

StringType type = STRING_REGULAR;

if (_peek(-1) == 'r') {

is_raw = true;

_advance();

} else if (_peek(-1) == '&') {

type = STRING_NAME;

_advance();

} else if (_peek(-1) == '^') {

type = STRING_NODEPATH;

_advance();

}

char32_t quote_char = _peek(-1);

if (_peek() == quote_char && _peek(1) == quote_char) {

is_multiline = true;

// Consume all quotes.

_advance();

_advance();

}

String result;

char32_t prev = 0;

int prev_pos = 0;

for (;;) {

// Consume actual string.

if (_is_at_end()) {

return make_error("Unterminated string.");

}

char32_t ch = _peek();

if (ch == 0x200E || ch == 0x200F || (ch >= 0x202A && ch <= 0x202E) || (ch >= 0x2066 && ch <= 0x2069)) {

Token error;

if (is_raw) {

error = make_error("Invisible text direction control character present in the string, use regular string literal instead of r-string.");

} else {

error = make_error("Invisible text direction control character present in the string, escape it (\"\\u" + String::num_int64(ch, 16) + "\") to avoid confusion.");

}

error.start_column = column;

error.end_column = column + 1;

push_error(error);

}

if (ch == '\\') {

// Escape pattern.

_advance();

if (_is_at_end()) {

return make_error("Unterminated string.");

}

if (is_raw) {

if (_peek() == quote_char) {

_advance();

if (_is_at_end()) {

return make_error("Unterminated string.");

}

result += '\\';

result += quote_char;

} else if (_peek() == '\\') { // For `\\\"`.

_advance();

if (_is_at_end()) {

return make_error("Unterminated string.");

}

result += '\\';

result += '\\';

} else {

result += '\\';

}

} else {

// Grab escape character.

char32_t code = _peek();

_advance();

if (_is_at_end()) {

return make_error("Unterminated string.");

}

char32_t escaped = 0;

bool valid_escape = true;

switch (code) {

case 'a':

escaped = '\a';

break;

case 'b':

escaped = '\b';

break;

case 'f':

escaped = '\f';

break;

case 'n':

escaped = '\n';

break;

case 'r':

escaped = '\r';

break;

case 't':

escaped = '\t';

break;

case 'v':

escaped = '\v';

break;

case '\'':

escaped = '\'';

break;

case '\"':

escaped = '\"';

break;

case '\\':

escaped = '\\';

break;

case 'U':

case 'u': {

// Hexadecimal sequence.

int hex_len = (code == 'U') ? 6 : 4;

for (int j = 0; j < hex_len; j++) {

if (_is_at_end()) {

return make_error("Unterminated string.");

}

char32_t digit = _peek();

char32_t value = 0;

if (is_digit(digit)) {

value = digit - '0';

} else if (digit >= 'a' && digit <= 'f') {

value = digit - 'a';

value += 10;

} else if (digit >= 'A' && digit <= 'F') {

value = digit - 'A';

value += 10;

} else {

// Make error, but keep parsing the string.

Token error = make_error("Invalid hexadecimal digit in unicode escape sequence.");

error.start_column = column;

error.end_column = column + 1;

push_error(error);

valid_escape = false;

break;

}

escaped <<= 4;