722 lines
25 KiB
Python
722 lines
25 KiB
Python
# ------------------------------------------------------------------------------
|
|
# pycparser: c_lexer.py
|
|
#
|
|
# CLexer class: lexer for the C language
|
|
#
|
|
# Eli Bendersky [https://eli.thegreenplace.net/]
|
|
# License: BSD
|
|
# ------------------------------------------------------------------------------
|
|
import re
|
|
from collections.abc import Callable
|
|
from dataclasses import dataclass
|
|
from enum import Enum
|
|
|
|
|
|
@dataclass(slots=True)
|
|
class Token:
|
|
type: str
|
|
value: str
|
|
lineno: int
|
|
column: int
|
|
|
|
|
|
class CLexer:
|
|
"""A standalone lexer for C.
|
|
|
|
The lexer takes the following callback functions as parameters during
|
|
construction:
|
|
error_func:
|
|
Called with (msg, line, column) on lexing errors.
|
|
on_lbrace_func:
|
|
Called when an LBRACE token is produced (used for scope tracking).
|
|
on_rbrace_func:
|
|
Called when an RBRACE token is produced (used for scope tracking).
|
|
type_lookup_func:
|
|
Called with an identifier name; expected to return True if it is
|
|
a typedef name and should be tokenized as TYPEID.
|
|
|
|
Call input(text) to initialize lexing, and then keep calling token() to
|
|
get the next token, until it returns None (at end of input).
|
|
"""
|
|
|
|
def __init__(
|
|
self,
|
|
error_func: Callable[[str, int, int], None],
|
|
on_lbrace_func: Callable[[], None],
|
|
on_rbrace_func: Callable[[], None],
|
|
type_lookup_func: Callable[[str], bool],
|
|
) -> None:
|
|
self.error_func = error_func
|
|
self.on_lbrace_func = on_lbrace_func
|
|
self.on_rbrace_func = on_rbrace_func
|
|
self.type_lookup_func = type_lookup_func
|
|
self._init_state()
|
|
|
|
def input(self, text: str, filename: str = "") -> None:
|
|
"""Initialize the lexer to the given input text.
|
|
|
|
filename is an optional name identifying the initial file from which the
|
|
input comes. The lexer may modify it if #line directives are
|
|
encountered.
|
|
"""
|
|
self._init_state()
|
|
self._lexdata = text
|
|
self._filename = filename
|
|
|
|
def _init_state(self) -> None:
|
|
self._lexdata = ""
|
|
self._filename = ""
|
|
self._pos = 0
|
|
self._line_start = 0
|
|
self._pending_tok: Token | None = None
|
|
self._lineno = 1
|
|
|
|
@property
|
|
def filename(self) -> str:
|
|
return self._filename
|
|
|
|
def token(self) -> Token | None:
|
|
# Lexing strategy overview:
|
|
#
|
|
# - We maintain a current position (self._pos), line number, and the
|
|
# byte offset of the current line start. The lexer is a simple loop
|
|
# that skips whitespace/newlines and emits one token per call.
|
|
#
|
|
# - A small amount of logic is handled manually before regex matching:
|
|
#
|
|
# * Preprocessor-style directives: if we see '#', we check whether
|
|
# it's a #line or #pragma directive and consume it inline. #line
|
|
# updates lineno/filename and produces no tokens. #pragma can yield
|
|
# both PPPRAGMA and PPPRAGMASTR, but token() returns a single token,
|
|
# so we stash the PPPRAGMASTR as _pending_tok to return on the next
|
|
# token() call. Otherwise we return PPHASH.
|
|
# * Newlines update lineno/line-start tracking so tokens can record
|
|
# accurate columns.
|
|
#
|
|
# - The bulk of tokens are recognized in _match_token:
|
|
#
|
|
# * _regex_rules: regex patterns for identifiers, literals, and other
|
|
# complex tokens (including error-producing patterns). The lexer
|
|
# uses a combined _regex_master to scan options at the same time.
|
|
# * _fixed_tokens: exact string matches for operators and punctuation,
|
|
# resolved by longest match.
|
|
#
|
|
# - Error patterns call the error callback and advance minimally, which
|
|
# keeps lexing resilient while reporting useful diagnostics.
|
|
text = self._lexdata
|
|
n = len(text)
|
|
|
|
if self._pending_tok is not None:
|
|
tok = self._pending_tok
|
|
self._pending_tok = None
|
|
return tok
|
|
|
|
while self._pos < n:
|
|
match text[self._pos]:
|
|
case " " | "\t":
|
|
self._pos += 1
|
|
case "\n":
|
|
self._lineno += 1
|
|
self._pos += 1
|
|
self._line_start = self._pos
|
|
case "#":
|
|
if _line_pattern.match(text, self._pos + 1):
|
|
self._pos += 1
|
|
self._handle_ppline()
|
|
continue
|
|
if _pragma_pattern.match(text, self._pos + 1):
|
|
self._pos += 1
|
|
toks = self._handle_pppragma()
|
|
if len(toks) > 1:
|
|
self._pending_tok = toks[1]
|
|
if len(toks) > 0:
|
|
return toks[0]
|
|
continue
|
|
tok = self._make_token("PPHASH", "#", self._pos)
|
|
self._pos += 1
|
|
return tok
|
|
case _:
|
|
if tok := self._match_token():
|
|
return tok
|
|
else:
|
|
continue
|
|
|
|
def _match_token(self) -> Token | None:
|
|
"""Match one token at the current position.
|
|
|
|
Returns a Token on success, or None if no token could be matched and
|
|
an error was reported. This method always advances _pos by the matched
|
|
length, or by 1 on error/no-match.
|
|
"""
|
|
text = self._lexdata
|
|
pos = self._pos
|
|
# We pick the longest match between:
|
|
# - the master regex (identifiers, literals, error patterns, etc.)
|
|
# - fixed operator/punctuator literals from the bucket for text[pos]
|
|
#
|
|
# The longest match is required to ensure we properly lex something
|
|
# like ".123" (a floating-point constant) as a single entity (with
|
|
# FLOAT_CONST), rather than a PERIOD followed by a number.
|
|
#
|
|
# The fixed-literal buckets are already length-sorted, so within that
|
|
# bucket we can take the first match. However, we still compare its
|
|
# length to the regex match because the regex may have matched a longer
|
|
# token that should take precedence.
|
|
best = None
|
|
|
|
if m := _regex_master.match(text, pos):
|
|
tok_type = m.lastgroup
|
|
# All master-regex alternatives are named; lastgroup shouldn't be None.
|
|
assert tok_type is not None
|
|
value = m.group(tok_type)
|
|
length = len(value)
|
|
action, msg = _regex_actions[tok_type]
|
|
best = (length, tok_type, value, action, msg)
|
|
|
|
if bucket := _fixed_tokens_by_first.get(text[pos]):
|
|
for entry in bucket:
|
|
if text.startswith(entry.literal, pos):
|
|
length = len(entry.literal)
|
|
if best is None or length > best[0]:
|
|
best = (
|
|
length,
|
|
entry.tok_type,
|
|
entry.literal,
|
|
_RegexAction.TOKEN,
|
|
None,
|
|
)
|
|
break
|
|
|
|
if best is None:
|
|
self._error(f"Illegal character {text[pos]!r}", pos)
|
|
self._pos += 1
|
|
return None
|
|
|
|
length, tok_type, value, action, msg = best
|
|
match action:
|
|
case _RegexAction.TOKEN:
|
|
pass
|
|
case _RegexAction.ERROR:
|
|
if tok_type == "BAD_CHAR_CONST":
|
|
msg = f"Invalid char constant {value}"
|
|
# All other ERROR rules provide a message.
|
|
assert msg is not None
|
|
self._error(msg, pos)
|
|
self._pos += max(1, length)
|
|
return None
|
|
case _RegexAction.ID:
|
|
tok_type = _keyword_map.get(value, "ID")
|
|
if tok_type == "ID" and self.type_lookup_func(value):
|
|
tok_type = "TYPEID"
|
|
case _:
|
|
raise RuntimeError("unreachable")
|
|
|
|
tok = self._make_token(tok_type, value, pos)
|
|
self._pos += length
|
|
|
|
if tok.type == "LBRACE":
|
|
self.on_lbrace_func()
|
|
elif tok.type == "RBRACE":
|
|
self.on_rbrace_func()
|
|
|
|
return tok
|
|
|
|
def _make_token(self, tok_type: str, value: str, pos: int) -> Token:
|
|
"""Create a Token at an absolute input position.
|
|
|
|
Expects tok_type/value and the absolute byte offset pos in the current
|
|
input. Does not advance lexer state; callers manage _pos themselves.
|
|
Returns a Token with lineno/column computed from current line tracking.
|
|
"""
|
|
column = pos - self._line_start + 1
|
|
tok = Token(tok_type, value, self._lineno, column)
|
|
return tok
|
|
|
|
def _error(self, msg: str, pos: int) -> None:
|
|
column = pos - self._line_start + 1
|
|
self.error_func(msg, self._lineno, column)
|
|
|
|
def _handle_ppline(self) -> None:
|
|
# Since #line directives aren't supposed to return tokens but should
|
|
# only affect the lexer's state (update line/filename for coords), this
|
|
# method does a bit of parsing on its own. It doesn't return anything,
|
|
# but its side effect is to update self._pos past the directive, and
|
|
# potentially update self._lineno and self._filename, based on the
|
|
# directive's contents.
|
|
#
|
|
# Accepted #line forms from preprocessors:
|
|
# - "#line 66 \"kwas\\df.h\""
|
|
# - "# 9"
|
|
# - "#line 10 \"include/me.h\" 1 2 3" (extra numeric flags)
|
|
# - "# 1 \"file.h\" 3"
|
|
# Errors we must report:
|
|
# - "#line \"file.h\"" (filename before line number)
|
|
# - "#line df" (garbage instead of number/string)
|
|
#
|
|
# We scan the directive line once (after an optional 'line' keyword),
|
|
# validating the order: NUMBER, optional STRING, then any NUMBERs.
|
|
# The NUMBERs tail is only accepted if a filename STRING was present.
|
|
text = self._lexdata
|
|
n = len(text)
|
|
line_end = text.find("\n", self._pos)
|
|
if line_end == -1:
|
|
line_end = n
|
|
line = text[self._pos : line_end]
|
|
pos = 0
|
|
line_len = len(line)
|
|
|
|
def skip_ws() -> None:
|
|
nonlocal pos
|
|
while pos < line_len and line[pos] in " \t":
|
|
pos += 1
|
|
|
|
skip_ws()
|
|
if line.startswith("line", pos):
|
|
pos += 4
|
|
|
|
def success(pp_line: str | None, pp_filename: str | None) -> None:
|
|
if pp_line is None:
|
|
self._error("line number missing in #line", self._pos + line_len)
|
|
else:
|
|
try:
|
|
self._lineno = int(pp_line)
|
|
except ValueError:
|
|
self._error("invalid #line directive", self._pos + line_len)
|
|
return
|
|
if pp_filename is not None:
|
|
self._filename = pp_filename
|
|
self._pos = line_end + 1
|
|
self._line_start = self._pos
|
|
|
|
def fail(msg: str, offset: int) -> None:
|
|
self._error(msg, self._pos + offset)
|
|
self._pos = line_end + 1
|
|
self._line_start = self._pos
|
|
|
|
skip_ws()
|
|
if pos >= line_len:
|
|
success(None, None)
|
|
return
|
|
if line[pos] == '"':
|
|
fail("filename before line number in #line", pos)
|
|
return
|
|
|
|
m = re.match(_decimal_constant, line[pos:])
|
|
if not m:
|
|
fail("invalid #line directive", pos)
|
|
return
|
|
|
|
pp_line = m.group(0)
|
|
pos += len(pp_line)
|
|
skip_ws()
|
|
if pos >= line_len:
|
|
success(pp_line, None)
|
|
return
|
|
|
|
if line[pos] != '"':
|
|
fail("invalid #line directive", pos)
|
|
return
|
|
|
|
m = re.match(_string_literal, line[pos:])
|
|
if not m:
|
|
fail("invalid #line directive", pos)
|
|
return
|
|
|
|
pp_filename = m.group(0).lstrip('"').rstrip('"')
|
|
pos += len(m.group(0))
|
|
|
|
# Consume arbitrary sequence of numeric flags after the directive
|
|
while True:
|
|
skip_ws()
|
|
if pos >= line_len:
|
|
break
|
|
m = re.match(_decimal_constant, line[pos:])
|
|
if not m:
|
|
fail("invalid #line directive", pos)
|
|
return
|
|
pos += len(m.group(0))
|
|
|
|
success(pp_line, pp_filename)
|
|
|
|
def _handle_pppragma(self) -> list[Token]:
|
|
# Parse a full #pragma line; returns a list of tokens with 1 or 2
|
|
# tokens - PPPRAGMA and an optional PPPRAGMASTR. If an empty list is
|
|
# returned, it means an error occurred, or we're at the end of input.
|
|
#
|
|
# Examples:
|
|
# - "#pragma" -> PPPRAGMA only
|
|
# - "#pragma once" -> PPPRAGMA, PPPRAGMASTR("once")
|
|
# - "# pragma omp parallel private(th_id)" -> PPPRAGMA, PPPRAGMASTR("omp parallel private(th_id)")
|
|
# - "#\tpragma {pack: 2, smack: 3}" -> PPPRAGMA, PPPRAGMASTR("{pack: 2, smack: 3}")
|
|
text = self._lexdata
|
|
n = len(text)
|
|
pos = self._pos
|
|
|
|
while pos < n and text[pos] in " \t":
|
|
pos += 1
|
|
if pos >= n:
|
|
self._pos = pos
|
|
return []
|
|
|
|
if not text.startswith("pragma", pos):
|
|
self._error("invalid #pragma directive", pos)
|
|
self._pos = pos + 1
|
|
return []
|
|
|
|
pragma_pos = pos
|
|
pos += len("pragma")
|
|
toks = [self._make_token("PPPRAGMA", "pragma", pragma_pos)]
|
|
|
|
while pos < n and text[pos] in " \t":
|
|
pos += 1
|
|
|
|
start = pos
|
|
while pos < n and text[pos] != "\n":
|
|
pos += 1
|
|
if pos > start:
|
|
toks.append(self._make_token("PPPRAGMASTR", text[start:pos], start))
|
|
if pos < n and text[pos] == "\n":
|
|
self._lineno += 1
|
|
pos += 1
|
|
self._line_start = pos
|
|
self._pos = pos
|
|
return toks
|
|
|
|
|
|
##
|
|
## Reserved keywords
|
|
##
|
|
_keywords: tuple[str, ...] = (
|
|
"AUTO",
|
|
"BREAK",
|
|
"CASE",
|
|
"CHAR",
|
|
"CONST",
|
|
"CONTINUE",
|
|
"DEFAULT",
|
|
"DO",
|
|
"DOUBLE",
|
|
"ELSE",
|
|
"ENUM",
|
|
"EXTERN",
|
|
"FLOAT",
|
|
"FOR",
|
|
"GOTO",
|
|
"IF",
|
|
"INLINE",
|
|
"INT",
|
|
"LONG",
|
|
"REGISTER",
|
|
"OFFSETOF",
|
|
"RESTRICT",
|
|
"RETURN",
|
|
"SHORT",
|
|
"SIGNED",
|
|
"SIZEOF",
|
|
"STATIC",
|
|
"STRUCT",
|
|
"SWITCH",
|
|
"TYPEDEF",
|
|
"UNION",
|
|
"UNSIGNED",
|
|
"VOID",
|
|
"VOLATILE",
|
|
"WHILE",
|
|
"__INT128",
|
|
"_BOOL",
|
|
"_COMPLEX",
|
|
"_GENERIC",
|
|
"_NORETURN",
|
|
"_THREAD_LOCAL",
|
|
"_STATIC_ASSERT",
|
|
"_ATOMIC",
|
|
"_ALIGNOF",
|
|
"_ALIGNAS",
|
|
"_PRAGMA",
|
|
)
|
|
|
|
_keyword_map: dict[str, str] = {}
|
|
|
|
for keyword in _keywords:
|
|
# Keywords from new C standard are mixed-case, like _Bool, _Alignas, etc.
|
|
if keyword.startswith("_") and len(keyword) > 1 and keyword[1].isalpha():
|
|
_keyword_map[keyword[:2].upper() + keyword[2:].lower()] = keyword
|
|
else:
|
|
_keyword_map[keyword.lower()] = keyword
|
|
|
|
##
|
|
## Regexes for use in tokens
|
|
##
|
|
|
|
# valid C identifiers (K&R2: A.2.3), plus '$' (supported by some compilers)
|
|
_identifier = r"[a-zA-Z_$][0-9a-zA-Z_$]*"
|
|
|
|
_hex_prefix = "0[xX]"
|
|
_hex_digits = "[0-9a-fA-F]+"
|
|
_bin_prefix = "0[bB]"
|
|
_bin_digits = "[01]+"
|
|
|
|
# integer constants (K&R2: A.2.5.1)
|
|
_integer_suffix_opt = (
|
|
r"(([uU]ll)|([uU]LL)|(ll[uU]?)|(LL[uU]?)|([uU][lL])|([lL][uU]?)|[uU])?"
|
|
)
|
|
_decimal_constant = (
|
|
"(0" + _integer_suffix_opt + ")|([1-9][0-9]*" + _integer_suffix_opt + ")"
|
|
)
|
|
_octal_constant = "0[0-7]*" + _integer_suffix_opt
|
|
_hex_constant = _hex_prefix + _hex_digits + _integer_suffix_opt
|
|
_bin_constant = _bin_prefix + _bin_digits + _integer_suffix_opt
|
|
|
|
_bad_octal_constant = "0[0-7]*[89]"
|
|
|
|
# comments are not supported
|
|
_unsupported_c_style_comment = r"\/\*"
|
|
_unsupported_cxx_style_comment = r"\/\/"
|
|
|
|
# Character constants and escape sequences.
|
|
#
|
|
# Tokens preserve their source spelling; the lexer does not decode escapes or
|
|
# validate Unicode code points. Each complete escape must match as one character
|
|
# in _cconst_char, which is used for both single and multicharacter constants.
|
|
# Matching is deliberately permissive: simple escapes accept extra letters and
|
|
# punctuation, and numeric escapes accept decimal digits, not just octal digits.
|
|
#
|
|
# _cconst_char is also repeated in the error patterns. If an escape can match in
|
|
# several ways, long malformed constants can cause exponential backtracking
|
|
# (https://github.com/eliben/pycparser/issues/61). The lookaheads prevent this:
|
|
# - Decimal and hexadecimal escapes consume the whole run of digits. For
|
|
# example, \123 cannot be split into \1 followed by the characters 2 and 3.
|
|
# - A simple \x escape is allowed only when no hexadecimal digit follows it.
|
|
# - A complete Unicode escape cannot fall back to a simple \u or \U escape.
|
|
# _bad_escape also excludes all decimal digits to agree with this permissiveness.
|
|
|
|
# Universal character names consume exactly four hex digits after u or eight
|
|
# after U. A following hex digit is a separate character, unlike with \x.
|
|
_unicode_escape = r"(u[0-9a-fA-F]{4}|U[0-9a-fA-F]{8})"
|
|
# This lookahead runs immediately after the backslash. If a complete Unicode
|
|
# escape follows, only _unicode_escape may match it: \u00e9 cannot be read as
|
|
# \u followed by 0, 0, e, 9. Incomplete forms retain the permissive behavior
|
|
# of simple escapes.
|
|
_simple_escape = (
|
|
rf"(?!{_unicode_escape})"
|
|
r"""([a-wyzA-Z._~!=&\^\-\\?'"]|x(?![0-9a-fA-F]))"""
|
|
)
|
|
_decimal_escape = r"""(\d+)(?!\d)"""
|
|
_hex_escape = r"""(x[0-9a-fA-F]+)(?![0-9a-fA-F])"""
|
|
_bad_escape = r"""([\\][^a-zA-Z._~^!=&\^\-\\?'"x0-9])"""
|
|
|
|
_escape_sequence = (
|
|
r"""(\\("""
|
|
+ _simple_escape
|
|
+ "|"
|
|
+ _decimal_escape
|
|
+ "|"
|
|
+ _hex_escape
|
|
+ "|"
|
|
+ _unicode_escape
|
|
+ "))"
|
|
)
|
|
|
|
# Strings do not need to count the characters represented by escapes. Match
|
|
# just the backslash and a permitted following character; _string_char consumes
|
|
# any remaining numeric or Unicode digits as ordinary text. This avoids matching
|
|
# complete escape sequences repeatedly. The permissive spellings also support
|
|
# Windows paths in #line filenames, which are matched with _string_literal.
|
|
_escape_sequence_start_in_string = r"""(\\[0-9a-zA-Z._~!=&\^\-\\?'"])"""
|
|
|
|
_cconst_char = r"""([^'\\\n]|""" + _escape_sequence + ")"
|
|
_char_const = "'" + _cconst_char + "'"
|
|
_wchar_const = "L" + _char_const
|
|
_u8char_const = "u8" + _char_const
|
|
_u16char_const = "u" + _char_const
|
|
_u32char_const = "U" + _char_const
|
|
_multicharacter_constant = "'" + _cconst_char + "{2,4}'"
|
|
_unmatched_quote = "('" + _cconst_char + "*\\n)|('" + _cconst_char + "*$)"
|
|
_bad_char_const = (
|
|
r"""('""" + _cconst_char + """[^'\n]+')|('')|('""" + _bad_escape + r"""[^'\n]*')"""
|
|
)
|
|
|
|
# String literals, including wide and Unicode prefixes.
|
|
_string_char = r"""([^"\\\n]|""" + _escape_sequence_start_in_string + ")"
|
|
_string_literal = '"' + _string_char + '*"'
|
|
_wstring_literal = "L" + _string_literal
|
|
_u8string_literal = "u8" + _string_literal
|
|
_u16string_literal = "u" + _string_literal
|
|
_u32string_literal = "U" + _string_literal
|
|
_bad_string_literal = '"' + _string_char + "*" + _bad_escape + _string_char + '*"'
|
|
|
|
# floating constants (K&R2: A.2.5.3)
|
|
_exponent_part = r"""([eE][-+]?[0-9]+)"""
|
|
_fractional_constant = r"""([0-9]*\.[0-9]+)|([0-9]+\.)"""
|
|
_floating_constant = (
|
|
"(((("
|
|
+ _fractional_constant
|
|
+ ")"
|
|
+ _exponent_part
|
|
+ "?)|([0-9]+"
|
|
+ _exponent_part
|
|
+ "))[FfLl]?)"
|
|
)
|
|
_binary_exponent_part = r"""([pP][+-]?[0-9]+)"""
|
|
_hex_fractional_constant = (
|
|
"(((" + _hex_digits + r""")?\.""" + _hex_digits + ")|(" + _hex_digits + r"""\.))"""
|
|
)
|
|
_hex_floating_constant = (
|
|
"("
|
|
+ _hex_prefix
|
|
+ "("
|
|
+ _hex_digits
|
|
+ "|"
|
|
+ _hex_fractional_constant
|
|
+ ")"
|
|
+ _binary_exponent_part
|
|
+ "[FfLl]?)"
|
|
)
|
|
|
|
|
|
class _RegexAction(Enum):
|
|
TOKEN = 0
|
|
ID = 1
|
|
ERROR = 2
|
|
|
|
|
|
@dataclass(frozen=True)
|
|
class _RegexRule:
|
|
# tok_type: name of the token emitted for a match
|
|
# regex_pattern: the raw regex (no anchors) to match at the current position
|
|
# action: TOKEN for normal tokens, ID for identifiers, ERROR to report
|
|
# error_message: message used for ERROR entries
|
|
tok_type: str
|
|
regex_pattern: str
|
|
action: _RegexAction
|
|
error_message: str | None
|
|
|
|
|
|
_regex_rules: list[_RegexRule] = [
|
|
_RegexRule(
|
|
"UNSUPPORTED_C_STYLE_COMMENT",
|
|
_unsupported_c_style_comment,
|
|
_RegexAction.ERROR,
|
|
"Comments are not supported, see https://github.com/eliben/pycparser#3using.",
|
|
),
|
|
_RegexRule(
|
|
"UNSUPPORTED_CXX_STYLE_COMMENT",
|
|
_unsupported_cxx_style_comment,
|
|
_RegexAction.ERROR,
|
|
"Comments are not supported, see https://github.com/eliben/pycparser#3using.",
|
|
),
|
|
_RegexRule(
|
|
"BAD_STRING_LITERAL",
|
|
_bad_string_literal,
|
|
_RegexAction.ERROR,
|
|
"String contains invalid escape code",
|
|
),
|
|
_RegexRule("WSTRING_LITERAL", _wstring_literal, _RegexAction.TOKEN, None),
|
|
_RegexRule("U8STRING_LITERAL", _u8string_literal, _RegexAction.TOKEN, None),
|
|
_RegexRule("U16STRING_LITERAL", _u16string_literal, _RegexAction.TOKEN, None),
|
|
_RegexRule("U32STRING_LITERAL", _u32string_literal, _RegexAction.TOKEN, None),
|
|
_RegexRule("STRING_LITERAL", _string_literal, _RegexAction.TOKEN, None),
|
|
_RegexRule("HEX_FLOAT_CONST", _hex_floating_constant, _RegexAction.TOKEN, None),
|
|
_RegexRule("FLOAT_CONST", _floating_constant, _RegexAction.TOKEN, None),
|
|
_RegexRule("INT_CONST_HEX", _hex_constant, _RegexAction.TOKEN, None),
|
|
_RegexRule("INT_CONST_BIN", _bin_constant, _RegexAction.TOKEN, None),
|
|
_RegexRule(
|
|
"BAD_CONST_OCT",
|
|
_bad_octal_constant,
|
|
_RegexAction.ERROR,
|
|
"Invalid octal constant",
|
|
),
|
|
_RegexRule("INT_CONST_OCT", _octal_constant, _RegexAction.TOKEN, None),
|
|
_RegexRule("INT_CONST_DEC", _decimal_constant, _RegexAction.TOKEN, None),
|
|
_RegexRule("INT_CONST_CHAR", _multicharacter_constant, _RegexAction.TOKEN, None),
|
|
_RegexRule("CHAR_CONST", _char_const, _RegexAction.TOKEN, None),
|
|
_RegexRule("WCHAR_CONST", _wchar_const, _RegexAction.TOKEN, None),
|
|
_RegexRule("U8CHAR_CONST", _u8char_const, _RegexAction.TOKEN, None),
|
|
_RegexRule("U16CHAR_CONST", _u16char_const, _RegexAction.TOKEN, None),
|
|
_RegexRule("U32CHAR_CONST", _u32char_const, _RegexAction.TOKEN, None),
|
|
_RegexRule("UNMATCHED_QUOTE", _unmatched_quote, _RegexAction.ERROR, "Unmatched '"),
|
|
_RegexRule("BAD_CHAR_CONST", _bad_char_const, _RegexAction.ERROR, None),
|
|
_RegexRule("ID", _identifier, _RegexAction.ID, None),
|
|
]
|
|
|
|
_regex_actions: dict[str, tuple[_RegexAction, str | None]] = {}
|
|
_regex_pattern_parts: list[str] = []
|
|
for _rule in _regex_rules:
|
|
_regex_actions[_rule.tok_type] = (_rule.action, _rule.error_message)
|
|
_regex_pattern_parts.append(f"(?P<{_rule.tok_type}>{_rule.regex_pattern})")
|
|
# The master regex is a single alternation of all token patterns, each wrapped
|
|
# in a named group. We match once at the current position and then use
|
|
# `lastgroup` to recover which token kind fired; this avoids iterating over all
|
|
# regexes on every character while keeping the same token-level semantics.
|
|
_regex_master: re.Pattern[str] = re.compile("|".join(_regex_pattern_parts))
|
|
|
|
|
|
@dataclass(frozen=True)
|
|
class _FixedToken:
|
|
tok_type: str
|
|
literal: str
|
|
|
|
|
|
_fixed_tokens: list[_FixedToken] = [
|
|
_FixedToken("ELLIPSIS", "..."),
|
|
_FixedToken("LSHIFTEQUAL", "<<="),
|
|
_FixedToken("RSHIFTEQUAL", ">>="),
|
|
_FixedToken("PLUSPLUS", "++"),
|
|
_FixedToken("MINUSMINUS", "--"),
|
|
_FixedToken("ARROW", "->"),
|
|
_FixedToken("LAND", "&&"),
|
|
_FixedToken("LOR", "||"),
|
|
_FixedToken("LSHIFT", "<<"),
|
|
_FixedToken("RSHIFT", ">>"),
|
|
_FixedToken("LE", "<="),
|
|
_FixedToken("GE", ">="),
|
|
_FixedToken("EQ", "=="),
|
|
_FixedToken("NE", "!="),
|
|
_FixedToken("TIMESEQUAL", "*="),
|
|
_FixedToken("DIVEQUAL", "/="),
|
|
_FixedToken("MODEQUAL", "%="),
|
|
_FixedToken("PLUSEQUAL", "+="),
|
|
_FixedToken("MINUSEQUAL", "-="),
|
|
_FixedToken("ANDEQUAL", "&="),
|
|
_FixedToken("OREQUAL", "|="),
|
|
_FixedToken("XOREQUAL", "^="),
|
|
_FixedToken("EQUALS", "="),
|
|
_FixedToken("PLUS", "+"),
|
|
_FixedToken("MINUS", "-"),
|
|
_FixedToken("TIMES", "*"),
|
|
_FixedToken("DIVIDE", "/"),
|
|
_FixedToken("MOD", "%"),
|
|
_FixedToken("OR", "|"),
|
|
_FixedToken("AND", "&"),
|
|
_FixedToken("NOT", "~"),
|
|
_FixedToken("XOR", "^"),
|
|
_FixedToken("LNOT", "!"),
|
|
_FixedToken("LT", "<"),
|
|
_FixedToken("GT", ">"),
|
|
_FixedToken("CONDOP", "?"),
|
|
_FixedToken("LPAREN", "("),
|
|
_FixedToken("RPAREN", ")"),
|
|
_FixedToken("LBRACKET", "["),
|
|
_FixedToken("RBRACKET", "]"),
|
|
_FixedToken("LBRACE", "{"),
|
|
_FixedToken("RBRACE", "}"),
|
|
_FixedToken("COMMA", ","),
|
|
_FixedToken("PERIOD", "."),
|
|
_FixedToken("SEMI", ";"),
|
|
_FixedToken("COLON", ":"),
|
|
]
|
|
|
|
# To avoid scanning all fixed tokens on every character, we bucket them by the
|
|
# first character. When matching at position i, we only look at the bucket for
|
|
# text[i], and we pre-sort that bucket by token length so the first match is
|
|
# also the longest. This preserves longest-match semantics (e.g. '>>=' before
|
|
# '>>' before '>') while reducing the number of comparisons.
|
|
_fixed_tokens_by_first: dict[str, list[_FixedToken]] = {}
|
|
for _entry in _fixed_tokens:
|
|
_fixed_tokens_by_first.setdefault(_entry.literal[0], []).append(_entry)
|
|
for _bucket in _fixed_tokens_by_first.values():
|
|
_bucket.sort(key=lambda item: len(item.literal), reverse=True)
|
|
|
|
_line_pattern: re.Pattern[str] = re.compile(r"([ \t]*line\W)|([ \t]*\d+)")
|
|
_pragma_pattern: re.Pattern[str] = re.compile(r"[ \t]*pragma\W")
|