NoobsGoHOE/rott
2
0
Fork 0
Code Issues Pull Requests Actions Packages Projects Releases Wiki Activity

Add missing tokens to lexer

Browse Source
This commit is contained in:
dkanus 2025-08-07 13:49:19 +07:00
parent 9ff20c7a60
commit 9ab65b0b02
1 changed files with 15 additions and 7 deletions
Show Stats Download Patch File Download Diff File Expand all files Collapse all files

22
rottlib/src/lexer/lexing.rs
View File

@ -2,7 +2,7 @@
//! //!
//! ## Notable details //! ## Notable details
//! //!
//! Lexer for UnrealScript that recognises inline `cpptext { … }` blocks. //! Lexer for UnrealScript that recognizes inline `cpptext { … }` blocks.
//! //!
//! In UnrealScript, `cpptext` lets authors embed raw C++ between braces. //! In UnrealScript, `cpptext` lets authors embed raw C++ between braces.
//! Because whitespace, newlines, or comments may appear between the //! Because whitespace, newlines, or comments may appear between the
@ -41,14 +41,14 @@ pub struct LexerState {
} }
/// Are these braces "real" UnrealScript braces, or the start/end of a C++ block? /// Are these braces "real" UnrealScript braces, or the start/end of a C++ block?
#[derive(Debug, PartialEq, Clone, Copy)] #[derive(Debug, PartialEq, Eq, Hash, Clone, Copy)]
pub enum BraceKind { pub enum BraceKind {
Normal, Normal,
CppBlock, CppBlock,
} }
/// All UnrealScript tokens that our compiler distinguishes. /// All UnrealScript tokens that our compiler distinguishes.
#[derive(logos::Logos, Debug, PartialEq, Clone, Copy)] #[derive(logos::Logos, Debug, PartialEq, Eq, Hash, Clone, Copy)]
#[logos(extras = LexerState)] #[logos(extras = LexerState)]
pub enum Token { pub enum Token {
// # Compiler/directive keywords // # Compiler/directive keywords
@ -247,9 +247,9 @@ pub enum Token {
#[token("~")] #[token("~")]
BitwiseNot, BitwiseNot,
// ## Vector // ## Vector
#[token("dot")] #[regex("(?i)dot")]
Dot, Dot,
#[token("cross")] #[regex("(?i)cross")]
Cross, Cross,
// ## Multiplicative // ## Multiplicative
#[token("*")] #[token("*")]
@ -290,6 +290,8 @@ pub enum Token {
NotEqual, NotEqual,
#[token("~=")] #[token("~=")]
ApproximatelyEqual, ApproximatelyEqual,
#[regex("(?i)clockwisefrom")]
ClockwiseFrom,
// ## Bitwise // ## Bitwise
#[token("&")] #[token("&")]
BitwiseAnd, BitwiseAnd,
@ -297,11 +299,11 @@ pub enum Token {
BitwiseOr, BitwiseOr,
#[token("^")] #[token("^")]
BitwiseXor, BitwiseXor,
#[token("^^")]
BooleanXor,
// ## Logical // ## Logical
#[token("&&")] #[token("&&")]
And, And,
#[token("^^")]
Xor,
#[token("||")] #[token("||")]
Or, Or,
// ## Assigments // ## Assigments
@ -311,6 +313,8 @@ pub enum Token {
MultiplyAssign, MultiplyAssign,
#[token("/=")] #[token("/=")]
DivideAssign, DivideAssign,
#[token("%=")]
ModuloAssign,
#[token("+=")] #[token("+=")]
PlusAssign, PlusAssign,
#[token("-=")] #[token("-=")]
@ -341,6 +345,10 @@ pub enum Token {
Period, Period,
#[token(":")] #[token(":")]
Colon, Colon,
#[token("#")]
Hash,
#[token("?")]
Question,
// # Comments & whitespaces // # Comments & whitespaces
#[regex(r"//[^\r\n]*")] #[regex(r"//[^\r\n]*")]