l0_lexer.py

l0_lexer.py

Module: l0_lexer

Source: compiler/stage1_py/l0_lexer.py Language: Python

Symbols

Namespace l0_lexer

Variable l0_lexer.KEYWORDS

1

dict l0_lexer.KEYWORDS

Variable l0_lexer._EXPR_ENDING_TOKENS

1

l0_lexer._EXPR_ENDING_TOKENS

Variable l0_lexer.OCT_CHARS

1

str l0_lexer.OCT_CHARS

Variable l0_lexer.HEX_CHARS

1

str l0_lexer.HEX_CHARS

Function l0_lexer._is_ascii_ident_start

1

bool l0_lexer._is_ascii_ident_start(str c)

Return whether c can start an ASCII Dea identifier.

Function l0_lexer._is_ascii_ident_part

1

bool l0_lexer._is_ascii_ident_part(str c)

Return whether c can continue an ASCII Dea identifier.

Function l0_lexer._logical_predecessor_ends_expression

1

bool l0_lexer._logical_predecessor_ends_expression(Token token, bool previous)

Return whether the parser-visible predecessor ends an expression.

Function l0_lexer.is_reserved_keyword

1

bool l0_lexer.is_reserved_keyword(str word)

Check if a word is a reserved L0 keyword.

Parameters:

  • word: The word to check.

Returns: True if the word is a keyword, False otherwise.

Class l0_lexer::TokenKind

Enumeration of all supported token types in L0.

Member Data l0_lexer.TokenKind.EOF

1

l0_lexer.TokenKind::EOF

Member Data l0_lexer.TokenKind.LEXER_ERROR

1

l0_lexer.TokenKind::LEXER_ERROR

Member Data l0_lexer.TokenKind.IDENT

1

l0_lexer.TokenKind::IDENT

Member Data l0_lexer.TokenKind.UNDERSCORE

1

l0_lexer.TokenKind::UNDERSCORE

Member Data l0_lexer.TokenKind.INT

1

l0_lexer.TokenKind::INT

Member Data l0_lexer.TokenKind.BYTE

1

l0_lexer.TokenKind::BYTE

Member Data l0_lexer.TokenKind.STRING

1

l0_lexer.TokenKind::STRING

Member Data l0_lexer.TokenKind.MODULE

1

l0_lexer.TokenKind::MODULE

Member Data l0_lexer.TokenKind.IMPORT

1

l0_lexer.TokenKind::IMPORT

Member Data l0_lexer.TokenKind.FUNC

1

l0_lexer.TokenKind::FUNC

Member Data l0_lexer.TokenKind.STRUCT

1

l0_lexer.TokenKind::STRUCT

Member Data l0_lexer.TokenKind.ENUM

1

l0_lexer.TokenKind::ENUM

Member Data l0_lexer.TokenKind.TYPE

1

l0_lexer.TokenKind::TYPE

Member Data l0_lexer.TokenKind.EXTERN

1

l0_lexer.TokenKind::EXTERN

Member Data l0_lexer.TokenKind.LET

1

l0_lexer.TokenKind::LET

Member Data l0_lexer.TokenKind.RETURN

1

l0_lexer.TokenKind::RETURN

Member Data l0_lexer.TokenKind.MATCH

1

l0_lexer.TokenKind::MATCH

Member Data l0_lexer.TokenKind.CASE

1

l0_lexer.TokenKind::CASE

Member Data l0_lexer.TokenKind.IF

1

l0_lexer.TokenKind::IF

Member Data l0_lexer.TokenKind.ELSE

1

l0_lexer.TokenKind::ELSE

Member Data l0_lexer.TokenKind.WHILE

1

l0_lexer.TokenKind::WHILE

Member Data l0_lexer.TokenKind.FOR

1

l0_lexer.TokenKind::FOR

Member Data l0_lexer.TokenKind.BREAK

1

l0_lexer.TokenKind::BREAK

Member Data l0_lexer.TokenKind.CONTINUE

1

l0_lexer.TokenKind::CONTINUE

Member Data l0_lexer.TokenKind.TRUE

1

l0_lexer.TokenKind::TRUE

Member Data l0_lexer.TokenKind.FALSE

1

l0_lexer.TokenKind::FALSE

Member Data l0_lexer.TokenKind.NULL

1

l0_lexer.TokenKind::NULL

Member Data l0_lexer.TokenKind.AS

1

l0_lexer.TokenKind::AS

Member Data l0_lexer.TokenKind.NEW

1

l0_lexer.TokenKind::NEW

Member Data l0_lexer.TokenKind.DROP

1

l0_lexer.TokenKind::DROP

Member Data l0_lexer.TokenKind.WITH

1

l0_lexer.TokenKind::WITH

Member Data l0_lexer.TokenKind.CLEANUP

1

l0_lexer.TokenKind::CLEANUP

Member Data l0_lexer.TokenKind.LBRACE

1

l0_lexer.TokenKind::LBRACE

Member Data l0_lexer.TokenKind.RBRACE

1

l0_lexer.TokenKind::RBRACE

Member Data l0_lexer.TokenKind.LPAREN

1

l0_lexer.TokenKind::LPAREN

Member Data l0_lexer.TokenKind.RPAREN

1

l0_lexer.TokenKind::RPAREN

Member Data l0_lexer.TokenKind.LBRACKET

1

l0_lexer.TokenKind::LBRACKET

Member Data l0_lexer.TokenKind.RBRACKET

1

l0_lexer.TokenKind::RBRACKET

Member Data l0_lexer.TokenKind.COMMA

1

l0_lexer.TokenKind::COMMA

Member Data l0_lexer.TokenKind.SEMI

1

l0_lexer.TokenKind::SEMI

Member Data l0_lexer.TokenKind.COLON

1

l0_lexer.TokenKind::COLON

Member Data l0_lexer.TokenKind.DOUBLE_COLON

1

l0_lexer.TokenKind::DOUBLE_COLON

Member Data l0_lexer.TokenKind.ARROW_FUNC

1

l0_lexer.TokenKind::ARROW_FUNC

Member Data l0_lexer.TokenKind.ARROW_MATCH

1

l0_lexer.TokenKind::ARROW_MATCH

Member Data l0_lexer.TokenKind.EQ

1

l0_lexer.TokenKind::EQ

Member Data l0_lexer.TokenKind.PLUS

1

l0_lexer.TokenKind::PLUS

Member Data l0_lexer.TokenKind.MINUS

1

l0_lexer.TokenKind::MINUS

Member Data l0_lexer.TokenKind.STAR

1

l0_lexer.TokenKind::STAR

Member Data l0_lexer.TokenKind.SLASH

1

l0_lexer.TokenKind::SLASH

Member Data l0_lexer.TokenKind.MODULO

1

l0_lexer.TokenKind::MODULO

Member Data l0_lexer.TokenKind.LT

1

l0_lexer.TokenKind::LT

Member Data l0_lexer.TokenKind.GT

1

l0_lexer.TokenKind::GT

Member Data l0_lexer.TokenKind.LE

1

l0_lexer.TokenKind::LE

Member Data l0_lexer.TokenKind.GE

1

l0_lexer.TokenKind::GE

Member Data l0_lexer.TokenKind.EQEQ

1

l0_lexer.TokenKind::EQEQ

Member Data l0_lexer.TokenKind.NE

1

l0_lexer.TokenKind::NE

Member Data l0_lexer.TokenKind.ANDAND

1

l0_lexer.TokenKind::ANDAND

Member Data l0_lexer.TokenKind.OROR

1

l0_lexer.TokenKind::OROR

Member Data l0_lexer.TokenKind.BANG

1

l0_lexer.TokenKind::BANG

Member Data l0_lexer.TokenKind.QUESTION

1

l0_lexer.TokenKind::QUESTION

Member Data l0_lexer.TokenKind.DOT

1

l0_lexer.TokenKind::DOT

Member Data l0_lexer.TokenKind.AMP

1

l0_lexer.TokenKind::AMP

Member Data l0_lexer.TokenKind.PIPE

1

l0_lexer.TokenKind::PIPE

Member Data l0_lexer.TokenKind.CARET

1

l0_lexer.TokenKind::CARET

Member Data l0_lexer.TokenKind.TILDE

1

l0_lexer.TokenKind::TILDE

Member Data l0_lexer.TokenKind.LSHIFT

1

l0_lexer.TokenKind::LSHIFT

Member Data l0_lexer.TokenKind.RSHIFT

1

l0_lexer.TokenKind::RSHIFT

Member Data l0_lexer.TokenKind.FUTURE_EXTENSION

1

l0_lexer.TokenKind::FUTURE_EXTENSION

Class l0_lexer::Token

A single token produced by the lexer.

Member Data l0_lexer.Token.kind

1

TokenKind kind

Member Data l0_lexer.Token.text

1

str text

Member Data l0_lexer.Token.line

1

int line

Member Data l0_lexer.Token.column

1

int column

Member Data l0_lexer.Token.diagnostic

1

Optional diagnostic

Member Data l0_lexer.Token.diagnostics

1

Optional diagnostics

Member Data l0_lexer.Token.recovery

1

Optional recovery

Function l0_lexer.Token.__repr__

1

str l0_lexer.Token.__repr__(self)

Returns a string representation of the token.

Class l0_lexer::Lexer

Tokenizes L0 source code.

This lexer scans source text and produces a list of Token objects. It tracks line and column information for diagnostics.

Member Data l0_lexer.Lexer.source

1

l0_lexer.Lexer::source

Member Data l0_lexer.Lexer.filename

1

l0_lexer.Lexer::filename

Member Data l0_lexer.Lexer.length

1

l0_lexer.Lexer::length

Member Data l0_lexer.Lexer.index

1

l0_lexer.Lexer::index

Member Data l0_lexer.Lexer.line

1

l0_lexer.Lexer::line

Member Data l0_lexer.Lexer.column

1

l0_lexer.Lexer::column

Member Data l0_lexer.Lexer.diagnostics

1

l0_lexer.Lexer::diagnostics

Member Data l0_lexer.Lexer._prev_ends_expression

1

l0_lexer.Lexer::_prev_ends_expression

Function l0_lexer.Lexer.__init__

1

None l0_lexer.Lexer.__init__(self, str source, str filename="<input>", Optional[List[Diagnostic]] diagnostics=None)

Initialize the lexer.

Parameters:

  • source: The source code text.
  • filename: The source filename for diagnostics.
  • diagnostics: Optional list to collect diagnostics into.

    Function l0_lexer.Lexer.from_source

1

"Lexer" l0_lexer.Lexer.from_source(cls, str source)

Create a lexer from a source string with default settings.

Parameters:

  • source: The source code text.

Returns: A new Lexer instance.

Function l0_lexer.Lexer.tokenize

1

List[Token] l0_lexer.Lexer.tokenize(self)

Perform tokenization of the source string.

Returns: A list of all tokens, ending with an EOF token.

Function l0_lexer.Lexer._error

1

None l0_lexer.Lexer._error(self, str message, int line, int column)

Add an error diagnostic.

Function l0_lexer.Lexer._lexer_error_token

1

Token l0_lexer.Lexer._lexer_error_token(self, str text, int line, int column, List[Diagnostic] diagnostics, Optional[Token] recovery=None)

Build a deferred lexer-error wrapper token.

Function l0_lexer.Lexer._wrap_new_diagnostics

1

Token l0_lexer.Lexer._wrap_new_diagnostics(self, int diagnostic_start, str text, int line, int column, Token recovery)

Move newly emitted recoverable diagnostics into a wrapper token.

Function l0_lexer.Lexer._at_end

1

bool l0_lexer.Lexer._at_end(self)

Check if all input has been scanned.

Function l0_lexer.Lexer._peek

1

str l0_lexer.Lexer._peek(self)

Peek at the current character without advancing.

Function l0_lexer.Lexer._peek_next

1

str l0_lexer.Lexer._peek_next(self)

Peek at the next character without advancing.

Function l0_lexer.Lexer._advance

1

str l0_lexer.Lexer._advance(self)

Advance the current index and return the character.

Function l0_lexer.Lexer._next_token

1

Token l0_lexer.Lexer._next_token(self)

Scan and return the next token.

Function l0_lexer.Lexer._read_byte_literal

1

str l0_lexer.Lexer._read_byte_literal(self, int start_col, int start_line)

Scan a byte/character literal.

Function l0_lexer.Lexer._read_string_literal

1

str l0_lexer.Lexer._read_string_literal(self)

Scan a double-quoted string literal.

Function l0_lexer.Lexer._read_valid_char_escape

1

str l0_lexer.Lexer._read_valid_char_escape(self)

Scan a single escape sequence.

Function l0_lexer.Lexer._read_number

1

str l0_lexer.Lexer._read_number(self, str c, int start_col, int start_line, bool is_negative=False)

Scan an integer literal.

Function l0_lexer.Lexer._skip_ws_and_comments

1

None l0_lexer.Lexer._skip_ws_and_comments(self)

Skip whitespace and both line and block comments.

Function l0_lexer.Lexer._collect_invalid_characters

1

str l0_lexer.Lexer._collect_invalid_characters(self)

Consume the rest of an invalid-character run and return its text.

The run stops at printable ASCII or whitespace, so a diagnostic span never crosses a line break.