NoobsGoHOE/rott
2
0
Fork 0
Code Issues Pull Requests Actions Packages Projects Releases Wiki Activity

Compare commits

base: NoobsGoHOE:f695f8a52e3b97b310628f2f65118a34f6a39146
Branches Tags
NoobsGoHOE:master
NoobsGoHOE:parsing_reimplementation
...
compare: NoobsGoHOE:797e5ea1927a2f6b0aae01637809356333968482
Branches Tags
NoobsGoHOE:parsing_reimplementation
NoobsGoHOE:master

2 Commits
f695f8a52e ... 797e5ea192

Author SHA1 Message Date
dkanus
797e5ea192 Transfer commit 2026-07-05 10:58:37 +07:00
dkanus
38de4a7419 Clean up expression parsing module 2026-05-03 02:43:23 +07:00
31 changed files with 2496 additions and 467 deletions
Show Stats Expand all files Collapse all files

226
dev_tests/src/verify_expr.rs
View File

@ -11,170 +11,136 @@ use rottlib::diagnostics::Diagnostic;
use rottlib::lexer::TokenizedFile;
use rottlib::parser::Parser;
const TEST_CASES: &[(&str, &str)] = &[
// P0057 - VarEditorSpecifierExpected
("files/P0057_01.uc", "var(Category,) int A;\n"),
("files/P0057_02.uc", "var(,) int A;\n"),
(
"files/P0057_03.uc",
"var(\n ,\n ,\n Category\n) int A;\n",
),
(
"files/P0057_04.uc",
"var(\n Category,\n\n\n) int A;\n",
),
// P0058 - VarEditorSpecifierListMissingSeparator
("files/P0058_01.uc", "var(Category DisplayName) int A;\n"),
("files/P0058_02.uc", "var(\"Display\" \"Tooltip\") int A;\n"),
(
"files/P0058_03.uc",
"var(\n Category\n DisplayName\n) int A;\n",
),
(
"files/P0058_04.uc",
"var(\n Category\n\n \"Display Name\",\n Advanced\n) int A;\n",
),
// P0059 - VarEditorSpecifierListExpectedCommaOrClosingParenthesis
("files/P0059_01.uc", "var(Category :) int A;\n"),
("files/P0059_02.uc", "var(\"Display\" *) int A;\n"),
(
"files/P0059_03.uc",
"var(\n Category\n [\n) int A;\n",
),
(
"files/P0059_04.uc",
"var(\n \"Display Name\"\n\n ]\n) int A;\n",
),
// P0060 - VarEditorSpecifierListMissingClosingParenthesis
("files/P0060_01.uc", "var(Category\n"),
(
"files/P0060_02.uc",
"var(\n Category,\n \"Display Name\"\n",
),
(
"files/P0060_03.uc",
"var(\n\n\n",
),
(
"files/P0060_04.uc",
"var(\n Category,\n\n \"Display Name\"\n\n",
),
];
/// Lexer-focused fixtures.
///
/// Keep these small: the goal is to inspect lexer diagnostics and delimiter
/// recovery behavior, not full parser behavior.
const TEST_CASES: &[(&str, &str)] = &[
// P0034 - SwitchMissingBody
("files/P0034_01.uc", "switch(A) local\n"),
("files/P0034_02.uc", "switch\n(A)\nvar"),
("files/P0034_03.uc", "switch(\n A\n)\n"),
("files/P0034_04.uc", "switch\n(\n A\n)\n"),
("files/P0034_05.uc", "switch(A)\ncase 1:\n"),
// P0035 - SwitchTopLevelItemNotCase
("files/P0035_01.uc", "switch(A) {\n Log(\"bad\");\n}\n"),
/*const TEST_CASES: &[(&str, &str)] = &[
// P0061 - TypeSpecifierExpected
("files/P0061_01.uc", "{\n local ;\n}\n"),
(
"files/P0035_02.uc",
"switch\n(A)\n{\n Log(\"bad\");\n Log(\"worse\");\n case 1:\n}\n",
),
("files/P0035_03.uc", "switch(A) {\n 123;\n default:\n}\n"),
(
"files/P0035_04.uc",
"switch\n(\n A\n)\n{\n if (A) {}\n case 1:\n}\n",
"files/P0061_02.uc",
"{\n local\n ;\n}\n",
),
(
"files/P0035_05.uc",
"switch(A) {\n {\n Log(\"nested\");\n }\n case 1:\n}\n",
"files/P0061_03.uc",
"{\n local\n [\n ] Values;\n}\n",
),
(
"files/P0061_04.uc",
"{\n local\n",
),
// P0036 - SwitchCaseMissingColon
// P0062 - NamedTypeInvalidName
("files/P0062_01.uc", "{\n local Foo. ;\n}\n"),
(
"files/P0036_01.uc",
"switch(A) {\n case 1\n case 2:\n}\n",
"files/P0062_02.uc",
"{\n local\n Foo.\n ;\n}\n",
),
(
"files/P0036_02.uc",
"switch\n(A)\n{\n case\n 1\n default:\n}\n",
"files/P0062_03.uc",
"{\n local Foo..Bar Value;\n}\n",
),
(
"files/P0036_03.uc",
"switch(A) {\n case (A)\n case B:\n}\n",
),
(
"files/P0036_04.uc",
"switch\n(\n A\n)\n{\n case\n A + B\n default\n :\n}\n",
),
(
"files/P0036_05.uc",
"switch(A) {\n case Foo.Bar(Baz)\n case Other:\n}\n",
"files/P0062_04.uc",
"{\n local\n SomePackage\n .\n",
),
// P0037 - SwitchDefaultMissingColon
// P0063 - ArrayTypeMissingOpeningAngleBracket
("files/P0063_01.uc", "{\n local array Values;\n}\n"),
(
"files/P0037_01.uc",
"switch(A) {\n default\n if (A) {}\n}\n",
"files/P0063_02.uc",
"{\n local\n array\n Values;\n}\n",
),
("files/P0063_03.uc", "{\n local array[] Values;\n}\n"),
(
"files/P0037_02.uc",
"switch\n(A)\n{\n default\n while (A) {}\n}\n",
),
(
"files/P0037_03.uc",
"switch(A) {\n default\n for (;;) {}\n}\n",
),
(
"files/P0037_04.uc",
"switch\n(\n A\n)\n{\n default\n switch(B) {\n case 1:\n }\n}\n",
),
(
"files/P0037_05.uc",
"switch(A) {\n default\n case 1:\n}\n",
"files/P0063_04.uc",
"{\n local\n array",
),
// P0038 - SwitchDuplicateDefault
// P0064 - ArrayTypeMissingClosingAngleBracket
("files/P0064_01.uc", "{\n local array<int Values;\n}\n"),
(
"files/P0038_01.uc",
"switch(A) {\n default:\n default:\n}\n",
"files/P0064_02.uc",
"{\n local\n array<\n int\n Values;\n}\n",
),
(
"files/P0038_02.uc",
"switch\n(A)\n{\n default\n :\n default\n :\n}\n",
"files/P0064_03.uc",
"{\n local array<class<Actor> ActorClasses;\n}\n",
),
(
"files/P0038_03.uc",
"switch(A) {\n default:\n default:\n default:\n}\n",
),
(
"files/P0038_04.uc",
"switch\n(\n A\n)\n{\n default:\n Log(\"first\");\n default:\n Log(\"second\");\n}\n",
"files/P0064_04.uc",
"{\n local\n array<\n string",
),
// P0039 - SwitchCasesAfterDefault
// P0065 - ArrayTypeMissingElementType
("files/P0065_01.uc", "{\n local array<> Values;\n}\n"),
(
"files/P0039_01.uc",
"switch(A) {\n default:\n case 1:\n}\n",
"files/P0065_02.uc",
"{\n local\n array<>\n Values;\n}\n",
),
(
"files/P0039_02.uc",
"switch\n(A)\n{\n default\n :\n case\n 1\n :\n}\n",
"files/P0065_03.uc",
"{\n local array<\n > Values;\n}\n",
),
(
"files/P0039_03.uc",
"switch(A) {\n default:\n case 1:\n case 2:\n}\n",
"files/P0065_04.uc",
"{\n local array<array<> > NestedValues;\n}\n",
),
(
"files/P0039_04.uc",
"switch\n(\n A\n)\n{\n default:\n case 1:\n Log(\"one\");\n case 2:\n Log(\"two\");\n}\n",
),
(
"files/P0039_05.uc",
"switch(A) {\n default:\n Log(\"done\");\n case 1:\n case 2:\n Log(\"stacked\");\n}\n",
),
// P0040 - SwitchMissingClosingBrace
("files/P0040_01.uc", "switch(A) {\n"),
("files/P0040_02.uc", "switch(A) {\n case 1:\n"),
("files/P0040_03.uc", "switch\n(A)\n{\n default:\n"),
(
"files/P0040_04.uc",
"switch\n(\n A\n)\n{\n case 1:\n case 2:\n",
),
(
"files/P0040_05.uc",
"switch(A) {\n case 1:\n Log(\"body\");\n",
),
// P0041 - SwitchCaseMissingExpression
("files/P0041_01.uc", "switch(A) {\n case:\n}\n"),
("files/P0041_02.uc", "switch\n(A)\n{\n case\n :\n}\n"),
("files/P0041_03.uc", "switch(A) {\n case:\n default:\n}\n"),
(
"files/P0041_04.uc",
"switch\n(\n A\n)\n{\n case\n :\n case 1:\n}\n",
),
(
"files/P0041_05.uc",
"switch(A) {\n case\n :\n default:\n}\n",
),
// P0042 - SwitchCaseExpressionInvalidStart
("files/P0042_01.uc", "switch(A) {\n case *:\n}\n"),
(
"files/P0042_02.uc",
"switch\n(A)\n{\n case\n =\n :\n}\n",
),
("files/P0042_03.uc", "switch(A) {\n case &&:\n default:\n}\n"),
(
"files/P0042_04.uc",
"switch\n(\n A\n)\n{\n case\n .\n :\n case 1:\n}\n",
),
(
"files/P0042_05.uc",
"switch(A) {\n case ]:\n}\n",
),
// Mixed / intentional cascades
(
"files/P0042_cascade_01.uc",
"switch(A) {\n case *\n case 1:\n}\n",
),
(
"files/P0042_cascade_02.uc",
"switch\n(\n A\n)\n{\n case\n =\n default:\n}\n",
),
];
];*/
/// If true, also run the parser after tokenization.
///

6
rottlib/src/ast/mod.rs
View File

@ -93,7 +93,7 @@ pub enum Statement<'src, 'arena> {
// `local int i, j, k`
LocalVariableDeclaration {
type_spec: TypeSpecifierRef<'src, 'arena>,
declarators: ArenaVec<'arena, VariableDeclaratorRef<'src, 'arena>>, // CHANGED
declarators: DeclaratorsList<'src, 'arena>, // CHANGED
},
Label(ArenaString<'arena>),
/// Nested function definitions inside blocks or states.
@ -192,7 +192,7 @@ pub struct ClassVarDecl<'src, 'arena> {
pub modifiers: ArenaVec<'arena, VarModifier>,
pub type_spec: TypeSpecifierRef<'src, 'arena>, // Named/InlineEnum/InlineStruct
pub declarators: ArenaVec<'arena, VariableDeclaratorRef<'src, 'arena>>, // a, b=expr
pub declarators: DeclaratorsList<'src, 'arena>, // a, b=expr
pub span: TokenSpan,
}
pub type ClassVarDeclRef<'src, 'arena> = ArenaNode<'arena, ClassVarDecl<'src, 'arena>>;
@ -259,7 +259,7 @@ pub struct StateDecl<'src, 'arena> {
pub modifiers: ArenaVec<'arena, StateModifier>, // auto, simulated
pub ignores: Option<ArenaVec<'arena, IdentifierToken>>, // 'ignores Foo, Bar;'
/// Body: ordinary statements plus nested function definitions (see `Statement::Function`).
pub body: ArenaVec<'arena, StatementRef<'src, 'arena>>,
pub body: StatementList<'src, 'arena>,
pub span: TokenSpan,
}
pub type StateDeclRef<'src, 'arena> = ArenaNode<'arena, StateDecl<'src, 'arena>>;

4
rottlib/src/ast/types.rs
View File

@ -114,7 +114,7 @@ pub struct StructField<'src, 'arena> {
/// Examples:
/// - `var int A;`
/// - `var int A, B[4], C = 10;`
pub declarators: ArenaVec<'arena, VariableDeclaratorRef<'src, 'arena>>,
pub declarators: DeclaratorsList<'src, 'arena>,
/// Optional `var(...)` editor specifiers attached to the field declaration.
///
/// Example:
@ -167,6 +167,8 @@ pub struct VariableDeclarator<'src, 'arena> {
/// The node span is expected to cover the entire declarator, not only the
/// identifier token.
pub type VariableDeclaratorRef<'src, 'arena> = ArenaNode<'arena, VariableDeclarator<'src, 'arena>>;
/// Declarations in `var` or `local` variable list.
pub type DeclaratorsList<'src, 'arena> = ArenaVec<'arena, VariableDeclaratorRef<'src, 'arena>>;
/// One item inside `var(...)` editor specifiers.
#[derive(Debug, PartialEq, Eq)]

441
rottlib/src/diagnostics/parse_error_diagnostics/declarations.rs Normal file
View File

@ -0,0 +1,441 @@
use super::{
Diagnostic, DiagnosticBuilder, FoundAt, found_at, primary_span_with_optional_multiline_context,
should_show_context_label,
};
use crate::lexer::{TokenSpan, TokenizedFile};
use crate::parser::{ParseError, diagnostic_labels};
const LOCAL_KEYWORD: &str = "local_keyword";
const DECLARATION_KEYWORD: &str = "declaration_keyword";
const VARIABLE_TYPE: &str = "variable_type";
const LAST_DECLARATOR: &str = "last_declarator";
const SEPARATOR: &str = "separator";
const OPEN_BRACKET: &str = "open_bracket";
fn related_span(error: &ParseError, key: &str) -> Option<TokenSpan> {
error.related_spans.get(key).copied()
}
fn should_label_keyword(
file: &TokenizedFile<'_>,
keyword_span: Option<TokenSpan>,
context_span: Option<TokenSpan>,
blame_span: TokenSpan,
) -> bool {
let Some(keyword_span) = keyword_span else {
return false;
};
let Some(context_span) = context_span else {
return !file.same_line(keyword_span.start, blame_span.end);
};
!file.same_line(keyword_span.start, context_span.end)
&& !file.same_line(keyword_span.start, blame_span.end)
}
fn expected_before_found(
file: &TokenizedFile<'_>,
blame_span: TokenSpan,
expected: &str,
) -> String {
match found_at(file, blame_span.start) {
FoundAt::Token(token_text) => format!("{expected} before `{token_text}`"),
FoundAt::EndOfFile => format!("{expected} before end of file"),
FoundAt::Unknown => format!("{expected} here"),
}
}
fn expected_variable_name_primary_text(file: &TokenizedFile<'_>, blame_span: TokenSpan) -> String {
expected_before_found(file, blame_span, "expected variable name")
}
/// P0050
pub(super) fn diagnostic_local_variable_declaration_missing_semicolon<'src>(
error: ParseError,
file: &TokenizedFile<'src>,
) -> Diagnostic {
let local_keyword_span = related_span(&error, LOCAL_KEYWORD);
let last_declarator_span = related_span(&error, LAST_DECLARATOR);
let blame_span = error.blame_span;
let primary_text = match found_at(file, blame_span.start) {
FoundAt::Token("local") => "expected `;` before this declaration".to_string(),
FoundAt::Token(token_text) => format!("expected `;` before `{token_text}`"),
FoundAt::EndOfFile => "expected `;` before end of file".to_string(),
FoundAt::Unknown => "expected `;` here".to_string(),
};
let mut builder = DiagnosticBuilder::error("missing `;` after local variable declaration");
if should_label_keyword(file, local_keyword_span, last_declarator_span, blame_span) {
if let Some(local_keyword_span) = local_keyword_span {
builder = builder
.secondary_label(local_keyword_span, "local variable declaration starts here");
}
}
if let Some(last_declarator_span) = last_declarator_span {
builder = builder.secondary_label(last_declarator_span, "declaration ends here");
}
builder
.primary_label(blame_span, primary_text)
.code("P0050")
.build()
}
/// P0051
pub(super) fn diagnostic_variable_declarator_list_empty<'src>(
error: ParseError,
file: &TokenizedFile<'src>,
) -> Diagnostic {
let declaration_keyword_span = related_span(&error, DECLARATION_KEYWORD);
let variable_type_span = related_span(&error, VARIABLE_TYPE);
let blame_span = error.blame_span;
let primary_text = expected_variable_name_primary_text(file, blame_span);
let mut builder =
DiagnosticBuilder::error("expected at least one variable name in variable declaration");
if should_label_keyword(
file,
declaration_keyword_span,
variable_type_span,
blame_span,
) {
if let Some(declaration_keyword_span) = declaration_keyword_span {
builder = builder
.secondary_label(declaration_keyword_span, "variable declaration starts here");
}
}
if let Some(variable_type_span) = variable_type_span {
if should_show_context_label(file, variable_type_span, blame_span) {
builder = builder.secondary_label(
variable_type_span,
"after this type, a variable name was expected",
);
}
}
builder
.primary_label(blame_span, primary_text)
.code("P0051")
.build()
}
/// P0052
pub(super) fn diagnostic_variable_declarator_expected<'src>(
error: ParseError,
file: &TokenizedFile<'src>,
) -> Diagnostic {
let declaration_keyword_span = related_span(&error, DECLARATION_KEYWORD);
let variable_type_span = related_span(&error, VARIABLE_TYPE);
let separator_span = related_span(&error, SEPARATOR);
let blame_span = error.blame_span;
let first_declarator = error.flags.contains("first_declarator");
let empty_declarator_run = error.flags.contains("empty_declarator_run");
let blame_is_multi_token = blame_span.start != blame_span.end;
let title = if first_declarator && empty_declarator_run {
"expected variable name before `,`"
} else if separator_span.is_some() {
"expected variable name after `,`"
} else {
"expected variable name in variable declaration"
};
let primary_text = if empty_declarator_run {
match found_at(file, blame_span.start) {
FoundAt::Token(",") if blame_is_multi_token => {
"empty variable declarators are not allowed".to_string()
}
FoundAt::Token(",") => "unexpected `,`".to_string(),
FoundAt::Token(token_text) => {
format!("expected variable name before `{token_text}`")
}
FoundAt::EndOfFile => "expected variable name before end of file".to_string(),
FoundAt::Unknown => "expected variable name here".to_string(),
}
} else if separator_span.is_some() {
match found_at(file, blame_span.start) {
FoundAt::Token(token_text) => {
format!("expected variable name before `{token_text}`")
}
FoundAt::EndOfFile => "expected variable name before end of file".to_string(),
FoundAt::Unknown => "expected variable name here".to_string(),
}
} else {
expected_variable_name_primary_text(file, blame_span)
};
let context_span = separator_span.or(variable_type_span);
let mut builder = DiagnosticBuilder::error(title);
if should_label_keyword(file, declaration_keyword_span, context_span, blame_span) {
if let Some(declaration_keyword_span) = declaration_keyword_span {
builder = builder
.secondary_label(declaration_keyword_span, "variable declaration starts here");
}
}
if let Some(separator_span) = separator_span {
if should_show_context_label(file, separator_span, blame_span) {
builder = builder.secondary_label(
separator_span,
"after this `,`, another variable name was expected",
);
}
} else if let Some(variable_type_span) = variable_type_span {
if should_show_context_label(file, variable_type_span, blame_span) {
builder = builder.secondary_label(
variable_type_span,
"after this type, a variable name was expected",
);
}
}
builder
.primary_label(blame_span, primary_text)
.code("P0052")
.build()
}
/// P0053
pub(super) fn diagnostic_variable_declarator_list_missing_separator<'src>(
error: ParseError,
file: &TokenizedFile<'src>,
) -> Diagnostic {
let declaration_keyword_span = related_span(&error, DECLARATION_KEYWORD);
let last_declarator_span = related_span(&error, LAST_DECLARATOR);
let blame_span = error.blame_span;
let primary_text = expected_before_found(file, blame_span, "expected `,`");
let mut builder = DiagnosticBuilder::error("missing `,` between variable declarators");
if should_label_keyword(
file,
declaration_keyword_span,
last_declarator_span,
blame_span,
) {
if let Some(declaration_keyword_span) = declaration_keyword_span {
builder = builder
.secondary_label(declaration_keyword_span, "variable declaration starts here");
}
}
if let Some(last_declarator_span) = last_declarator_span {
builder = builder.secondary_label(last_declarator_span, "previous declarator ends here");
}
builder
.primary_label(blame_span, primary_text)
.code("P0053")
.build()
}
/// P0054
pub(super) fn diagnostic_variable_declarator_array_size_missing_closing_bracket<'src>(
error: ParseError,
file: &TokenizedFile<'src>,
) -> Diagnostic {
let declaration_keyword_span = related_span(&error, DECLARATION_KEYWORD);
let open_bracket_span = related_span(&error, OPEN_BRACKET);
let blame_span = error.blame_span;
let primary_text = expected_before_found(file, blame_span, "expected `]`");
let primary_span =
primary_span_with_optional_multiline_context(file, open_bracket_span, blame_span);
let mut builder = DiagnosticBuilder::error("missing `]` to close array size expression");
if should_label_keyword(
file,
declaration_keyword_span,
open_bracket_span,
blame_span,
) {
if let Some(declaration_keyword_span) = declaration_keyword_span {
builder = builder
.secondary_label(declaration_keyword_span, "variable declaration starts here");
}
}
if let Some(open_bracket_span) = open_bracket_span {
if should_show_context_label(file, open_bracket_span, blame_span) {
builder =
builder.secondary_label(open_bracket_span, "array size expression starts here");
}
}
builder
.primary_label(primary_span, primary_text)
.code("P0054")
.build()
}
/// P0055
pub(super) fn diagnostic_variable_declarator_array_size_expected<'src>(
error: ParseError,
file: &TokenizedFile<'src>,
) -> Diagnostic {
let variable_name_span = related_span(
&error,
diagnostic_labels::EXPRESSION_REQUIRED_BY,
);
let expected_after_span = related_span(
&error,
diagnostic_labels::EXPRESSION_EXPECTED_AFTER,
);
let blame_span = error.blame_span;
let variable_name = variable_name_span.and_then(|span| match found_at(file, span.start) {
FoundAt::Token(token_text) => Some(token_text),
FoundAt::EndOfFile | FoundAt::Unknown => None,
});
let title = match variable_name {
Some(variable_name) => {
format!("expected array size expression for `{variable_name}` between `[` and `]`")
}
None => "expected array size expression between `[` and `]`".to_string(),
};
let primary_text = match found_at(file, blame_span.start) {
FoundAt::Token(token_text) => {
format!("expected array size expression before `{token_text}`")
}
FoundAt::EndOfFile => "expected array size expression before end of file".to_string(),
FoundAt::Unknown => "expected array size expression here".to_string(),
};
let primary_span =
primary_span_with_optional_multiline_context(file, expected_after_span, blame_span);
let mut builder = DiagnosticBuilder::error(title);
if let Some(expected_after_span) = expected_after_span {
if should_show_context_label(file, expected_after_span, blame_span) {
builder = builder.secondary_label(
expected_after_span,
"after this `[`, an array size expression was expected",
);
}
}
if let Some(variable_name_span) = variable_name_span {
let variable_name_is_already_visible =
file.same_line(variable_name_span.start, blame_span.start)
|| file.same_line(variable_name_span.start, blame_span.end)
|| expected_after_span.is_some_and(|expected_after_span| {
file.same_line(variable_name_span.start, expected_after_span.start)
|| file.same_line(variable_name_span.start, expected_after_span.end)
});
if should_show_context_label(file, variable_name_span, blame_span)
&& !variable_name_is_already_visible
{
builder = builder.secondary_label(
variable_name_span,
"this variable has an array size suffix",
);
}
}
builder
.primary_label(primary_span, primary_text)
.code("P0055")
.build()
}
/// P0056
pub(super) fn diagnostic_variable_declarator_initializer_expected<'src>(
error: ParseError,
file: &TokenizedFile<'src>,
) -> Diagnostic {
let variable_name_span = related_span(
&error,
diagnostic_labels::EXPRESSION_REQUIRED_BY,
);
let expected_after_span = related_span(
&error,
diagnostic_labels::EXPRESSION_EXPECTED_AFTER,
);
let blame_span = error.blame_span;
let variable_name = variable_name_span.and_then(|span| match found_at(file, span.start) {
FoundAt::Token(token_text) => Some(token_text),
FoundAt::EndOfFile | FoundAt::Unknown => None,
});
let title = match (variable_name, found_at(file, blame_span.start)) {
(Some(variable_name), FoundAt::Token(token_text)) => {
format!(
"expected initializer expression for `{variable_name}` after `=`, found `{token_text}`"
)
}
(Some(variable_name), FoundAt::EndOfFile) => {
format!(
"expected initializer expression for `{variable_name}` after `=`, found end of file"
)
}
(Some(variable_name), FoundAt::Unknown) => {
format!("expected initializer expression for `{variable_name}` after `=`")
}
(None, FoundAt::Token(token_text)) => {
format!("expected initializer expression after `=`, found `{token_text}`")
}
(None, FoundAt::EndOfFile) => {
"expected initializer expression after `=`, found end of file".to_string()
}
(None, FoundAt::Unknown) => "expected initializer expression after `=`".to_string(),
};
let primary_text = match found_at(file, blame_span.start) {
FoundAt::Token(token_text) => format!("unexpected `{token_text}`"),
FoundAt::EndOfFile => "reached end of file here".to_string(),
FoundAt::Unknown => "expected initializer expression here".to_string(),
};
let mut builder = DiagnosticBuilder::error(title);
if let Some(expected_after_span) = expected_after_span {
if should_show_context_label(file, expected_after_span, blame_span) {
builder = builder.secondary_label(
expected_after_span,
"after this `=`, an initializer expression was expected",
);
}
}
if let Some(variable_name_span) = variable_name_span {
let variable_name_is_already_visible =
file.same_line(variable_name_span.start, blame_span.start)
|| file.same_line(variable_name_span.start, blame_span.end)
|| expected_after_span.is_some_and(|expected_after_span| {
file.same_line(variable_name_span.start, expected_after_span.start)
|| file.same_line(variable_name_span.start, expected_after_span.end)
});
if should_show_context_label(file, variable_name_span, blame_span)
&& !variable_name_is_already_visible
{
builder = builder.secondary_label(
variable_name_span,
"initializer is for this variable",
);
}
}
builder
.primary_label(blame_span, primary_text)
.code("P0056")
.build()
}

24
rottlib/src/diagnostics/parse_error_diagnostics/mod.rs
View File

@ -19,6 +19,7 @@ use crate::parser::{ParseError, ParseErrorKind};
mod block_items;
mod control_flow_expressions;
mod declarations;
mod primary_expressions;
mod selector_expressions;
mod switch_expressions;
@ -77,6 +78,7 @@ pub(crate) fn diagnostic_from_parse_error<'src>(
file: &TokenizedFile<'src>,
) -> Diagnostic {
use control_flow_expressions::*;
use declarations::*;
use primary_expressions::*;
use selector_expressions::*;
use switch_expressions::*;
@ -189,6 +191,28 @@ pub(crate) fn diagnostic_from_parse_error<'src>(
ParseErrorKind::SwitchCaseExpressionInvalidStart => {
diagnostic_switch_case_expression_invalid_start(error, file)
}
// declarations.rs
ParseErrorKind::LocalVariableDeclarationMissingSemicolon => {
diagnostic_local_variable_declaration_missing_semicolon(error, file)
}
ParseErrorKind::VariableDeclaratorListEmpty => {
diagnostic_variable_declarator_list_empty(error, file)
}
ParseErrorKind::VariableDeclaratorExpected => {
diagnostic_variable_declarator_expected(error, file)
}
ParseErrorKind::VariableDeclaratorListMissingSeparator => {
diagnostic_variable_declarator_list_missing_separator(error, file)
}
ParseErrorKind::VariableDeclaratorArraySizeMissingClosingBracket => {
diagnostic_variable_declarator_array_size_missing_closing_bracket(error, file)
}
ParseErrorKind::VariableDeclaratorArraySizeExpected => {
diagnostic_variable_declarator_array_size_expected(error, file)
}
ParseErrorKind::VariableDeclaratorInitializerExpected => {
diagnostic_variable_declarator_initializer_expected(error, file)
}
_ => DiagnosticBuilder::error(format!("error {:?} while parsing", error.kind))
.primary_label(error.covered_span, "happened here")

7
rottlib/src/diagnostics/parse_error_diagnostics/primary_expressions.rs
View File

@ -1,6 +1,6 @@
use super::{Diagnostic, DiagnosticBuilder, FoundAt, collapse_span_to_end_on_same_line, found_at};
use crate::lexer::{TokenSpan, TokenizedFile};
use crate::parser::ParseError;
use crate::parser::{ParseError, diagnostic_labels};
pub(super) fn diagnostic_parenthesized_expression_invalid_start<'src>(
error: ParseError,
@ -164,7 +164,10 @@ pub(super) fn diagnostic_class_type_expected_qualified_type_name<'src>(
error: ParseError,
file: &TokenizedFile<'src>,
) -> Diagnostic {
let qualifier_dot_span = error.related_spans.get("qualifier_dot").copied();
let qualifier_dot_span = error
.related_spans
.get(diagnostic_labels::QUALIFIED_IDENTIFIER_DOT)
.copied();
let class_span = error.related_spans.get("class_keyword").copied();
let blame_pos = error.blame_span.end;

6
rottlib/src/lexer/token.rs
View File

@ -1,10 +1,10 @@
//! Token definitions for Fermented `UnrealScript`.
//! Token definitions for Fermented UnrealScript.
//!
//! These are the tokens consumed by the parser and derived from [`RawToken`]s.
use super::{BraceKind, raw_lexer::RawToken};
/// Tokens consumed by the Fermented `UnrealScript` parser.
/// Tokens consumed by the Fermented UnrealScript parser.
#[derive(Clone, Copy, Debug, Hash, PartialEq, Eq)]
pub enum Token {
ExecDirective,
@ -408,7 +408,7 @@ impl Token {
}
}
/// Reserved words of Fermented `UnrealScript`.
/// Reserved words of Fermented UnrealScript.
///
/// These are represented in [`Token`] as [`Token::Keyword`].
#[derive(Clone, Copy, Debug, Hash, PartialEq, Eq)]

3
rottlib/src/parser/cursor.rs
View File

@ -321,7 +321,7 @@ impl<'src, 'arena> Parser<'src, 'arena> {
self.advance();
Ok(token_position)
} else {
let anchor = self.peek_position().unwrap_or_else(|| self.file.eof());
let anchor = self.peek_position_or_eof();
Err(self
.make_error_at(error_kind, anchor)
.blame(TokenSpan::new(anchor)))
@ -378,6 +378,7 @@ impl<'src, 'arena> Parser<'src, 'arena> {
"parser made no forward progress"
);
if peeked_position <= old_position {
// TODO: check if we're in error and panic if not
self.advance();
}
if self.file.is_eof(&old_position) {

68
rottlib/src/parser/errors.rs
View File

@ -1,6 +1,6 @@
//! Submodule with parsing related errors.
use std::collections::HashMap;
use std::collections::{HashMap, HashSet};
use crate::{lexer::TokenPosition, lexer::TokenSpan};
@ -100,27 +100,63 @@ pub enum ParseErrorKind {
SwitchCaseMissingExpression,
/// P0042
SwitchCaseExpressionInvalidStart,
/// P0043
InvalidNumericLiteral,
/// P0044
EnumMissingName,
/// P0045
EnumMissingOpeningBrace,
/// P0046
EnumMissingClosingBrace,
/// P0047
EnumExpectedCommaOrClosingBraceAfterVariant,
/// P0048
EnumVariantInvalidStart,
/// P0049
EnumExpectedVariant,
/// P0050
LocalVariableDeclarationMissingSemicolon,
/// P0051
VariableDeclaratorListEmpty,
/// P0052
VariableDeclaratorExpected,
/// P0053
VariableDeclaratorListMissingSeparator,
/// P0054
VariableDeclaratorArraySizeMissingClosingBracket,
/// P0055
VariableDeclaratorArraySizeExpected,
/// P0056
VariableDeclaratorInitializerExpected,
/// P0057
VarEditorSpecifierExpected,
/// P0058
VarEditorSpecifierListMissingSeparator,
/// P0059
VarEditorSpecifierListExpectedCommaOrClosingParenthesis,
/// P0060
VarEditorSpecifierListMissingClosingParenthesis,
/// P0061
TypeSpecifierExpected,
/// P0062
NamedTypeInvalidName,
/// P0063
ArrayTypeMissingOpeningAngleBracket,
/// P0064
ArrayTypeMissingClosingAngleBracket,
/// P0065
ArrayTypeMissingElementType,
// ================== Old errors to be thrown away! ==================
/// Found an unexpected token while parsing an expression.
ExpressionUnexpectedToken,
DeclEmptyVariableDeclarations,
DeclNoSeparatorBetweenVariableDeclarations,
DeclExpectedRightBracketAfterArraySize,
DeclExpectedCommaAfterVariableDeclarator,
TypeSpecExpectedType,
TypeSpecInvalidNamedTypeName,
TypeSpecArrayMissingOpeningAngle,
TypeSpecArrayMissingInnerType,
TypeSpecArrayMissingClosingAngle,
TypeSpecClassMissingInnerType,
TypeSpecClassMissingClosingAngle,
BlockMissingSemicolonAfterStatement,
/// Unexpected end of input while parsing.
UnexpectedEndOfFile,
/// Token looked like a numeric literal but could not be parsed as one.
InvalidNumericLiteral,
/// A bare expression appeared in a `switch` arm but was not the final arm.
///
/// Such an expression must be terminated with `;` or be the final arm.
@ -140,8 +176,6 @@ pub enum ParseErrorKind {
///
/// At least one variable name must follow the type.
DeclMissingIdentifier,
/// Invalid variable name identifier in non-`local` variable definition.
DeclBadVariableIdentifier,
/// Found an unexpected token while parsing a declaration literal.
///
/// Expected one of: integer, float, string, `true`, `false`, `none`
@ -206,15 +240,12 @@ pub enum ParseErrorKind {
FunctionParamsMissingOpeningParenthesis,
FunctionParamsMissingClosingParenthesis,
ClassUnexpectedItem,
EnumMissingLeftBrace,
EnumBadVariant,
StructFieldMissingName,
StructFieldMissingSemicolon,
StructMissingRightBrace,
// Named enum/struct typedefs
EnumMissingKeyword, // class member: expected `enum`
EnumExpectedNameOrBrace, // after `enum`, expected identifier
EnumNoClosingBrace,
EnumMissingKeyword, // class member: expected `enum`
EnumEmptyVariants,
EnumNoSeparatorBetweenVariants,
EnumMissingLBrace,
@ -275,6 +306,7 @@ pub enum ParseErrorKind {
StateParensMissingRParen,
BadTypeInClassTypeDeclaration,
IdentifierExpected,
VariableDeclaratorBadIdentifier,
// --- Generic list diagnostics (comma-separated, closed by `)`) ---
/// Saw `)` immediately after `(`, or closed the list without any items.
@ -338,6 +370,7 @@ pub struct ParseError {
/// The source span in which the error was detected.
pub covered_span: TokenSpan,
pub related_spans: HashMap<String, TokenSpan>,
pub flags: HashSet<String>,
}
pub type ParseResult<'src, 'arena, T> = Result<T, ParseError>;
@ -358,6 +391,7 @@ impl crate::parser::Parser<'_, '_> {
blame_span: TokenSpan::new(position),
covered_span: TokenSpan::new(position),
related_spans: HashMap::new(),
flags: HashSet::new(),
}
}
}

17
rottlib/src/parser/grammar/class.rs
View File

@ -12,7 +12,7 @@ use crate::ast::{
use crate::lexer::{Keyword, Token, TokenPosition, TokenSpan};
use crate::parser::{ParseErrorKind, ParseResult, ResultRecoveryExt, SyncLevel};
use std::collections::HashMap;
use std::collections::{HashMap, HashSet};
impl<'src, 'arena> crate::parser::Parser<'src, 'arena> {
#[inline]
@ -744,7 +744,7 @@ impl<'src, 'arena> crate::parser::Parser<'src, 'arena> {
let is_negative = matches!(token, Token::Minus);
self.advance();
let (next_token, next_lexeme, _) =
let (next_token, next_lexeme, token_position) =
self.require_token_lexeme_and_position(ParseErrorKind::InvalidNumericLiteral)?;
match next_token {
@ -761,7 +761,7 @@ impl<'src, 'arena> crate::parser::Parser<'src, 'arena> {
let mut signed_lexeme = String::with_capacity(1 + next_lexeme.len());
signed_lexeme.push(if is_negative { '-' } else { '+' });
signed_lexeme.push_str(next_lexeme);
let value = self.decode_float_literal(&signed_lexeme)?;
let value = self.decode_float_literal(&signed_lexeme, token_position)?;
self.advance();
DeclarationLiteral::Float(value)
}
@ -778,7 +778,7 @@ impl<'src, 'arena> crate::parser::Parser<'src, 'arena> {
DeclarationLiteral::Integer(value)
}
Token::FloatLiteral => {
let value = self.decode_float_literal(lexeme)?;
let value = self.decode_float_literal(lexeme, token_position)?;
self.advance();
DeclarationLiteral::Float(value)
}
@ -844,8 +844,8 @@ impl<'src, 'arena> crate::parser::Parser<'src, 'arena> {
ParseErrorKind::ClassUnexpectedItem,
)?;
let name = self.parse_identifier(ParseErrorKind::DeclBadVariableIdentifier)?;
self.expect(Token::Assign, ParseErrorKind::TypeSpecInvalidNamedTypeName)?;
let name = self.parse_identifier(ParseErrorKind::VariableDeclaratorBadIdentifier)?;
self.expect(Token::Assign, ParseErrorKind::NamedTypeInvalidName)?;
let value = self.parse_declaration_literal_class()?;
self.expect(Token::Semicolon, ParseErrorKind::DeclMissingSemicolon)?;
@ -865,7 +865,7 @@ impl<'src, 'arena> crate::parser::Parser<'src, 'arena> {
match self.peek_token_and_position() {
Some((next_token, declarator_start)) if next_token.is_valid_identifier_name() => {
let identifier = self
.parse_identifier(ParseErrorKind::DeclBadVariableIdentifier)
.parse_identifier(ParseErrorKind::VariableDeclaratorBadIdentifier)
.unwrap_or(IdentifierToken(declarator_start));
let array_size = match self.parse_array_len_expr() {
@ -898,7 +898,7 @@ impl<'src, 'arena> crate::parser::Parser<'src, 'arena> {
break;
}
Some((_, _)) if declarators.is_empty() => {
self.report_error_here(ParseErrorKind::DeclBadVariableIdentifier);
self.report_error_here(ParseErrorKind::VariableDeclaratorBadIdentifier);
self.recover_until(SyncLevel::StatementStart);
let _ = self.eat(Token::Semicolon);
break;
@ -951,6 +951,7 @@ impl<'src, 'arena> crate::parser::Parser<'src, 'arena> {
blame_span: TokenSpan::range(list_start, list_end),
covered_span: TokenSpan::range(list_start, list_end),
related_spans: HashMap::new(),
flags: HashSet::new(),
});
}

35
rottlib/src/parser/grammar/declarations/enum_definition.rs
View File

@ -1,11 +1,11 @@
//! Parsing of enum definitions for Fermented `UnrealScript`.
//! Parsing of enum definitions for Fermented UnrealScript.
use std::ops::ControlFlow;
use crate::arena::ArenaVec;
use crate::ast::{EnumDefRef, EnumDefinition, IdentifierToken};
use crate::lexer::Token;
use crate::lexer::{TokenSpan, TokenPosition};
use crate::lexer::{TokenPosition, TokenSpan};
use crate::parser::{ParseErrorKind, Parser, ResultRecoveryExt, SyncLevel};
#[derive(Clone, Copy, Debug, Hash, PartialEq, Eq)]
@ -24,12 +24,19 @@ impl<'src, 'arena> Parser<'src, 'arena> {
enum_keyword_position: TokenPosition,
) -> EnumDefRef<'src, 'arena> {
let name = self
.parse_identifier(ParseErrorKind::EnumExpectedNameOrBrace)
.parse_identifier(ParseErrorKind::EnumMissingName)
.unwrap_or_fallback(self);
self.expect(Token::LeftBrace, ParseErrorKind::EnumMissingLeftBrace)
self.expect(Token::LeftBrace, ParseErrorKind::EnumMissingOpeningBrace)
.related_token("enum_keyword", enum_keyword_position)
.report_error(self);
let variants = self.parse_enum_variants();
self.expect(Token::RightBrace, ParseErrorKind::EnumNoClosingBrace)
if variants.is_empty() {
self.make_error_at_last_consumed(ParseErrorKind::EnumExpectedVariant)
.related_token("enum_keyword", enum_keyword_position)
.report_error(self);
}
self.expect(Token::RightBrace, ParseErrorKind::EnumMissingClosingBrace)
.related_token("enum_keyword", enum_keyword_position)
.report_error(self);
let span = TokenSpan::range(
@ -88,6 +95,10 @@ impl<'src, 'arena> Parser<'src, 'arena> {
}
self.make_error_at_last_consumed(ParseErrorKind::EnumEmptyVariants)
.widen_error_span_from(error_start_position)
.blame(TokenSpan::range(
error_start_position,
self.last_consumed_position_or_start(),
))
.report_error(self);
if matches!(self.peek_token(), Some(Token::RightBrace) | None) {
ControlFlow::Break(())
@ -103,7 +114,7 @@ impl<'src, 'arena> Parser<'src, 'arena> {
&mut self,
variants: &mut ArenaVec<'arena, IdentifierToken>,
) -> ControlFlow<()> {
self.parse_identifier(ParseErrorKind::EnumBadVariant)
self.parse_identifier(ParseErrorKind::EnumVariantInvalidStart)
.sync_error_until(self, SyncLevel::StatementStart)
.ok_or_report(self)
.map_or(ControlFlow::Break(()), |variant| {
@ -120,7 +131,7 @@ impl<'src, 'arena> Parser<'src, 'arena> {
variants: &mut ArenaVec<'arena, IdentifierToken>,
) -> ControlFlow<()> {
let Some(variant) = self
.parse_identifier(ParseErrorKind::EnumBadVariant)
.parse_identifier(ParseErrorKind::EnumExpectedCommaOrClosingBraceAfterVariant)
.widen_error_span_from(error_start_position)
.sync_error_until(self, SyncLevel::StatementStart)
.ok_or_report(self)
@ -128,9 +139,13 @@ impl<'src, 'arena> Parser<'src, 'arena> {
// If we don't even get a good identifier - error is different
return ControlFlow::Break(());
};
self.make_error_at_last_consumed(ParseErrorKind::EnumNoSeparatorBetweenVariants)
.widen_error_span_from(error_start_position)
.report_error(self);
let mut error = self.make_error_at_last_consumed(ParseErrorKind::EnumNoSeparatorBetweenVariants)
.widen_error_span_from(error_start_position);
if let Some(previous_variant) = variants.last() {
error = error.related("previous_variant", previous_variant.span());
}
error.report_error(self);
variants.push(variant);
ControlFlow::Continue(())

2
rottlib/src/parser/grammar/declarations/mod.rs
View File

@ -1,4 +1,4 @@
//! Declaration parsing for Fermented `UnrealScript`.
//! Declaration parsing for Fermented UnrealScript.
//!
//! Implements recursive-descent parsing for declaration-related grammar:
//! type specifiers, enum and struct definitions, `var(...)` prefixes,

8
rottlib/src/parser/grammar/declarations/struct_definition.rs
View File

@ -1,8 +1,8 @@
//! Parsing of struct definitions for Fermented `UnrealScript`.
//! Parsing of struct definitions for Fermented UnrealScript.
//!
//! ## C++ block handling
//!
//! The Fermented `UnrealScript` parser must support parsing several legacy
//! The Fermented UnrealScript parser must support parsing several legacy
//! source files that contain `cpptext` or `cppstruct`. Our compiler does not
//! compile with C++ code and therefore does not need these blocks in
//! the resulting AST. We treat them the same as trivia and skip them.
@ -120,7 +120,9 @@ impl<'src, 'arena> Parser<'src, 'arena> {
let Some(field_prefix) = self.parse_struct_field_prefix() else {
return StructBodyItemParseOutcome::Skip;
};
let declarators = self.parse_variable_declarators();
// TODO: correct span
let declarators = self
.parse_variable_declarators(var_keyword_position, TokenSpan::new(var_keyword_position));
if !self.eat(Token::Semicolon) {
self.report_error_here(ParseErrorKind::StructFieldMissingSemicolon);
self.recover_until(SyncLevel::BlockBoundary);

72
rottlib/src/parser/grammar/declarations/type_specifier.rs
View File

@ -1,8 +1,8 @@
//! Parsing of type specifiers for Fermented `UnrealScript`.
//! Parsing of type specifiers for Fermented UnrealScript.
use crate::ast::{TypeSpecifier, TypeSpecifierRef};
use crate::lexer::{Keyword, Token, TokenPosition, TokenSpan};
use crate::parser::{ParseErrorKind, ParseResult, Parser};
use crate::parser::{ParseErrorKind, ParseResult, Parser, ResultRecoveryExt, SyncLevel};
impl<'src, 'arena> Parser<'src, 'arena> {
/// Parses a type specifier used in variable declarations.
@ -15,7 +15,7 @@ impl<'src, 'arena> Parser<'src, 'arena> {
&mut self,
) -> ParseResult<'src, 'arena, TypeSpecifierRef<'src, 'arena>> {
let (starting_token, starting_token_position) =
self.require_token_and_position(ParseErrorKind::TypeSpecExpectedType)?;
self.require_token_and_position(ParseErrorKind::TypeSpecifierExpected)?;
match starting_token {
Token::Keyword(Keyword::Enum) => {
@ -36,13 +36,16 @@ impl<'src, 'arena> Parser<'src, 'arena> {
}
_ if starting_token.is_valid_type_name() => {
let type_name =
self.parse_qualified_identifier(ParseErrorKind::TypeSpecInvalidNamedTypeName)?;
self.parse_qualified_identifier(ParseErrorKind::NamedTypeInvalidName)?;
let full_span = *type_name.span();
Ok(self
.arena
.alloc_node(TypeSpecifier::Named(type_name), full_span))
}
_ => Err(self.make_error_at_last_consumed(ParseErrorKind::TypeSpecExpectedType)),
_ => Err(self.make_error_at(
ParseErrorKind::TypeSpecifierExpected,
starting_token_position,
)),
}
}
@ -70,18 +73,41 @@ impl<'src, 'arena> Parser<'src, 'arena> {
&mut self,
starting_token_position: TokenPosition,
) -> ParseResult<'src, 'arena, TypeSpecifierRef<'src, 'arena>> {
self.expect(
let left_angle_bracket = self.expect(
Token::Less,
ParseErrorKind::TypeSpecArrayMissingOpeningAngle,
ParseErrorKind::ArrayTypeMissingOpeningAngleBracket,
)?;
if self.eat(Token::Greater) {
return Err(
self.make_error_at_last_consumed(ParseErrorKind::ArrayTypeMissingElementType)
);
}
let element_modifiers = self.parse_var_declaration_modifiers();
let element_type = self.parse_type_specifier()?;
let closing_angle_bracket_position = self.expect(
// Don't sync on this error, because for a better recovery in likely
// case of missing `>` we need to check whether inserting it would fix
// the issue.
let element_type = self.parse_type_specifier().unwrap_or_fallback(self);
// A type-closing `>` can be followed up by either another `>` or
// identifier, so on failure check whether we have an identifier next.
let closing_angle_bracket_position = match self.expect(
Token::Greater,
ParseErrorKind::TypeSpecArrayMissingClosingAngle,
)?;
let array_span = TokenSpan::range(starting_token_position, closing_angle_bracket_position);
ParseErrorKind::ArrayTypeMissingClosingAngleBracket,
) {
Ok(position) => position,
Err(error) => {
let next_token_can_start_declarator = self
.peek_token()
.is_some_and(|token| token.is_valid_identifier_name());
let error = if next_token_can_start_declarator {
error
} else {
error.sync_error_at(self, SyncLevel::CloseAngleBracket)
};
Err(error).unwrap_or_fallback(self)
}
};
let array_span = TokenSpan::range(starting_token_position, closing_angle_bracket_position);
Ok(self.arena.alloc_node(
TypeSpecifier::Array {
element_type,
@ -95,18 +121,16 @@ impl<'src, 'arena> Parser<'src, 'arena> {
&mut self,
starting_token_position: TokenPosition,
) -> ParseResult<'src, 'arena, TypeSpecifierRef<'src, 'arena>> {
let (inner_type_name, class_type_end) = if self.eat(Token::Less) {
let inner_type_name = Some(
self.parse_qualified_identifier(ParseErrorKind::TypeSpecClassMissingInnerType)?,
);
let class_type_end = self.expect(
Token::Greater,
ParseErrorKind::TypeSpecClassMissingClosingAngle,
)?;
(inner_type_name, class_type_end)
} else {
(None, starting_token_position)
};
let (inner_type_name, class_type_end) =
if let Some(left_angle_bracket_position) = self.eat_with_position(Token::Less) {
let parsed = self.parse_class_type_argument_tail(
starting_token_position,
left_angle_bracket_position,
)?;
(Some(parsed.type_name), parsed.right_angle_bracket_position)
} else {
(None, starting_token_position)
};
let span = TokenSpan::range(starting_token_position, class_type_end);
Ok(self
.arena

262
rottlib/src/parser/grammar/declarations/var_specifiers.rs
View File

@ -1,11 +1,56 @@
//! Parsing of declaration specifiers used in `var(...) ...` syntax for
//! Fermented `UnrealScript`.
//! Fermented UnrealScript.
use std::ops::ControlFlow;
use crate::arena::ArenaVec;
use crate::ast::{VarEditorSpecifier, VarEditorSpecifierRef, VarModifier};
use crate::lexer::Token;
use crate::lexer::{Token, TokenPosition, TokenSpan};
use crate::parser::{ParseErrorKind, Parser, ResultRecoveryExt, SyncLevel};
#[derive(Clone, Copy, Debug, Hash, PartialEq, Eq)]
enum VarEditorSpecifierListParseStep {
ExpectingSpecifier,
ExpectingSeparator,
}
struct VarEditorSpecifierListParseState<'arena, SpecifierRef> {
editor_specifiers: ArenaVec<'arena, SpecifierRef>,
open_parenthesis_position: TokenPosition,
pending_separator_position: Option<TokenPosition>,
previous_specifier_span: Option<TokenSpan>,
parse_step: VarEditorSpecifierListParseStep,
}
impl<'arena, SpecifierRef> VarEditorSpecifierListParseState<'arena, SpecifierRef> {
#[must_use]
fn new<'src>(parser: &Parser<'src, 'arena>, open_parenthesis_position: TokenPosition) -> Self {
Self {
editor_specifiers: parser.arena.vec(),
open_parenthesis_position,
pending_separator_position: None,
previous_specifier_span: None,
parse_step: VarEditorSpecifierListParseStep::ExpectingSpecifier,
}
}
fn set_specifier_expected(&mut self, separator_position: TokenPosition) {
self.pending_separator_position = Some(separator_position);
self.parse_step = VarEditorSpecifierListParseStep::ExpectingSpecifier;
}
fn set_separator_expected(&mut self) {
self.pending_separator_position = None;
self.parse_step = VarEditorSpecifierListParseStep::ExpectingSeparator;
}
fn push_specifier(&mut self, specifier: SpecifierRef, specifier_span: TokenSpan) {
self.previous_specifier_span = Some(specifier_span);
self.editor_specifiers.push(specifier);
self.set_separator_expected();
}
}
impl<'src, 'arena> Parser<'src, 'arena> {
/// Parses a consecutive run of variable declaration modifiers.
///
@ -20,70 +65,201 @@ impl<'src, 'arena> Parser<'src, 'arena> {
#[must_use]
pub(crate) fn parse_var_declaration_modifiers(&mut self) -> ArenaVec<'arena, VarModifier> {
let mut modifiers = self.arena.vec();
while let Some(current_token_and_position) = self.peek_token_and_position() {
let Ok(parsed_modifier) = VarModifier::try_from(current_token_and_position) else {
break;
};
while let Some(current_token_and_position) = self.peek_token_and_position()
&& let Ok(parsed_modifier) = VarModifier::try_from(current_token_and_position)
{
self.advance();
modifiers.push(parsed_modifier);
}
modifiers
}
// TODO: should this have "optional" in the name?
/// Parses the optional parenthesized editor specifier list in `var(...)`.
///
/// Assumes that `var` has already been consumed.
///
/// Returns `None` if the current token is not `(`. Returns `Some(...)` once
/// `(` is present, including for an empty list.
///
/// Recovery is intentionally minimal because these specifier lists are not
/// important enough to justify aggressive repair.
#[must_use]
pub(crate) fn parse_var_editor_specifier_list(
&mut self,
) -> Option<ArenaVec<'arena, VarEditorSpecifierRef<'src, 'arena>>> {
if !self.eat(Token::LeftParenthesis) {
use VarEditorSpecifierListParseStep::{ExpectingSeparator, ExpectingSpecifier};
let Some(left_parenthesis_position) = self.eat_with_position(Token::LeftParenthesis) else {
return None;
}
let mut editor_specifiers = self.arena.vec();
while let Some((next_token, next_token_lexeme, next_token_position)) =
self.peek_token_lexeme_and_position()
&& next_token != Token::RightParenthesis
{
if next_token == Token::StringLiteral {
self.advance();
let string_value = self.unescape_string_literal(next_token_lexeme);
editor_specifiers.push(self.arena.alloc_node_at(
VarEditorSpecifier::String(string_value),
next_token_position,
));
} else if let Some(specifier_identifier) =
Self::identifier_token_from_token(next_token, next_token_position)
{
self.advance();
editor_specifiers.push(self.arena.alloc_node_at(
VarEditorSpecifier::Identifier(specifier_identifier),
next_token_position,
));
} else {
self.make_error_at_last_consumed(ParseErrorKind::VarSpecNotIdentifier)
.sync_error_until(self, SyncLevel::ListSeparator)
.report_error(self);
}
// Detailed recovery is not worthwhile here;
// stop once list structure becomes unclear.
if !self.eat(Token::Comma) {
break;
};
let mut state = VarEditorSpecifierListParseState::new(self, left_parenthesis_position);
// Progress invariant: every branch that continues the loop consumes at
// least one token. Branches that only report an error must stop the
// loop or synchronize before continuing.
while let Some((next_token, next_token_position)) = self.peek_token_and_position() {
match (state.parse_step, next_token) {
(ExpectingSpecifier, Token::RightParenthesis) => {
if state.pending_separator_position.is_some() {
self.report_missing_var_editor_specifier_at_current_token(&state);
}
break;
}
(ExpectingSpecifier, Token::Comma) => {
if self
.recover_empty_var_editor_specifier_run(next_token_position, &mut state)
.is_break()
{
break;
}
}
(ExpectingSpecifier, _) => {
if let Some(parsed_specifier) = self.try_parse_current_var_editor_specifier() {
let parsed_specifier_span = *parsed_specifier.span();
state.push_specifier(parsed_specifier, parsed_specifier_span);
} else {
self.report_missing_var_editor_specifier_at_current_token(&state);
break;
}
}
(ExpectingSeparator, Token::Comma) => {
self.advance(); // ','
state.set_specifier_expected(next_token_position);
}
(ExpectingSeparator, Token::RightParenthesis) => break,
(ExpectingSeparator, _) => {
if let Some(parsed_specifier) = self.try_parse_current_var_editor_specifier() {
let parsed_specifier_span = *parsed_specifier.span();
self.make_error_at_last_consumed(
ParseErrorKind::VarEditorSpecifierListMissingSeparator,
)
.widen_error_span_from(next_token_position)
.related_token("open_parenthesis", state.open_parenthesis_position)
.maybe_related("previous_specifier", state.previous_specifier_span)
.report_error(self);
state.push_specifier(parsed_specifier, parsed_specifier_span);
} else {
self.report_unexpected_token_after_var_editor_specifier(&state);
break;
}
}
}
// Guard against parser bugs that would otherwise leave declaration
// parsing stuck on the same token.
self.ensure_forward_progress(next_token_position);
}
self.expect(
Token::RightParenthesis,
ParseErrorKind::VarSpecsMissingClosingParenthesis,
ParseErrorKind::VarEditorSpecifierListMissingClosingParenthesis,
)
.sync_error_at(self, SyncLevel::CloseParenthesis)
.related_token("open_parenthesis", state.open_parenthesis_position)
.maybe_related("previous_specifier", state.previous_specifier_span)
.sync_error_at_matching_delimiter(self, left_parenthesis_position)
.report_error(self);
Some(state.editor_specifiers)
}
/// Tries to parse the current token as one `var(...)` editor specifier.
///
/// This is the only recognizer for editor specifier starts. It returns
/// [`None`] without consuming anything when the current token cannot start a
/// specifier.
fn try_parse_current_var_editor_specifier(
&mut self,
) -> Option<VarEditorSpecifierRef<'src, 'arena>> {
let (next_token, next_token_lexeme, next_token_position) =
self.peek_token_lexeme_and_position()?;
if next_token == Token::StringLiteral {
let string_value = self.unescape_string_literal(next_token_lexeme);
self.advance();
return Some(self.arena.alloc_node_at(
VarEditorSpecifier::String(string_value),
next_token_position,
));
}
let specifier_identifier =
Self::identifier_token_from_token(next_token, next_token_position)?;
self.advance();
Some(self.arena.alloc_node_at(
VarEditorSpecifier::Identifier(specifier_identifier),
next_token_position,
))
}
/// Reports a missing editor specifier at the current token.
fn report_missing_var_editor_specifier_at_current_token(
&mut self,
state: &VarEditorSpecifierListParseState<'arena, VarEditorSpecifierRef<'src, 'arena>>,
) {
let position = self.peek_position_or_eof();
let missing_first_specifier =
state.editor_specifiers.is_empty() && state.pending_separator_position.is_none();
let mut error = self
.make_error_at(ParseErrorKind::VarEditorSpecifierExpected, position)
.related_token("open_parenthesis", state.open_parenthesis_position)
.maybe_related_token("separator", state.pending_separator_position)
.maybe_related("previous_specifier", state.previous_specifier_span);
if missing_first_specifier {
error = error.with_flag("first_specifier");
}
error.report(self);
}
/// Recovers from one or more empty editor specifier slots produced by
/// commas.
///
/// Called when the current token is `,`. Returns [`ControlFlow::Continue`]
/// only when either the list can continue or a following specifier was
/// consumed as part of recovery.
fn recover_empty_var_editor_specifier_run(
&mut self,
error_start_position: TokenPosition,
state: &mut VarEditorSpecifierListParseState<'arena, VarEditorSpecifierRef<'src, 'arena>>,
) -> ControlFlow<()> {
let missing_first_specifier =
state.editor_specifiers.is_empty() && state.pending_separator_position.is_none();
while let Some(Token::Comma) = self.peek_token() {
self.advance();
}
let mut error = self
.make_error_at_last_consumed(ParseErrorKind::VarEditorSpecifierExpected)
.widen_error_span_from(error_start_position)
.extend_blame_start_to_covered_start()
.related_token("open_parenthesis", state.open_parenthesis_position)
.maybe_related_token("separator", state.pending_separator_position)
.maybe_related("previous_specifier", state.previous_specifier_span)
.with_flag("empty_specifier_run");
if missing_first_specifier {
error = error.with_flag("first_specifier");
}
error.report_error(self);
if self.peek_token() == Some(Token::RightParenthesis) {
return ControlFlow::Break(());
}
if let Some(parsed_specifier) = self.try_parse_current_var_editor_specifier() {
let parsed_specifier_span = *parsed_specifier.span();
state.push_specifier(parsed_specifier, parsed_specifier_span);
ControlFlow::Continue(())
} else {
ControlFlow::Break(())
}
}
/// Reports a token that appears after an editor specifier where only `,`
/// or `)` can continue the list.
fn report_unexpected_token_after_var_editor_specifier(
&mut self,
state: &VarEditorSpecifierListParseState<'arena, VarEditorSpecifierRef<'src, 'arena>>,
) {
let position = self.peek_position_or_eof();
self.make_error_at(
ParseErrorKind::VarEditorSpecifierListExpectedCommaOrClosingParenthesis,
position,
)
.related_token("open_parenthesis", state.open_parenthesis_position)
.maybe_related("previous_specifier", state.previous_specifier_span)
.sync_error_until(self, SyncLevel::CloseParenthesis)
.report_error(self);
Some(editor_specifiers)
}
}

295
rottlib/src/parser/grammar/declarations/variable_declarators.rs
View File

@ -1,172 +1,345 @@
//! Parsing of comma-separated variable declarator lists for
//! Fermented `UnrealScript`.
//! Fermented UnrealScript.
//!
//! Extends original `UnrealScript` by allowing array-size expressions and
//! declarator initializers.
//! The Fermented dialect extends original UnrealScript with array-size
//! expressions and declarator initializers.
#![allow(clippy::option_if_let_else)]
use std::ops::ControlFlow;
use crate::arena::ArenaVec;
use crate::ast::{OptionalExpression, VariableDeclarator, VariableDeclaratorRef};
use crate::ast::{self, OptionalExpression, VariableDeclarator, VariableDeclaratorRef};
use crate::lexer::{Token, TokenPosition, TokenSpan};
use crate::parser::{ParseErrorKind, ParseResult, Parser, ResultRecoveryExt, SyncLevel};
#[derive(Clone, Copy, Debug, Hash, PartialEq, Eq)]
enum VariableDeclaratorParseState {
enum VariableDeclaratorListParseStep {
ExpectingDeclarator,
ExpectingSeparator,
}
#[derive(Debug)]
struct VariableDeclaratorListParseState<'src, 'arena> {
declarators: ast::DeclaratorsList<'src, 'arena>,
declaration_keyword_position: TokenPosition,
variable_type_span: TokenSpan,
pending_separator_position: Option<TokenPosition>,
previous_declarator_span: Option<TokenSpan>,
parse_step: VariableDeclaratorListParseStep,
}
impl<'src, 'arena> VariableDeclaratorListParseState<'src, 'arena> {
#[must_use]
fn new(
parser: &Parser<'src, 'arena>,
declaration_keyword_position: TokenPosition,
variable_type_span: TokenSpan,
) -> Self {
Self {
declarators: parser.arena.vec(),
declaration_keyword_position,
variable_type_span,
pending_separator_position: None,
previous_declarator_span: None,
parse_step: VariableDeclaratorListParseStep::ExpectingDeclarator,
}
}
fn set_declarator_expected(&mut self, separator_position: TokenPosition) {
self.pending_separator_position = Some(separator_position);
self.parse_step = VariableDeclaratorListParseStep::ExpectingDeclarator;
}
fn set_separator_expected(&mut self) {
self.pending_separator_position = None;
self.parse_step = VariableDeclaratorListParseStep::ExpectingSeparator;
}
fn push_declarator(&mut self, declarator: VariableDeclaratorRef<'src, 'arena>) {
self.previous_declarator_span = Some(*declarator.span());
self.declarators.push(declarator);
self.set_separator_expected();
}
fn missing_declarator_error_kind(&self) -> ParseErrorKind {
if self.declarators.is_empty() {
ParseErrorKind::VariableDeclaratorListEmpty
} else {
ParseErrorKind::VariableDeclaratorExpected
}
}
}
impl<'src, 'arena> Parser<'src, 'arena> {
/// Parses a comma-separated list of variable declarators.
/// Parses a comma-separated variable declarator list after a declaration
/// keyword and type.
///
/// Accepts optional array-size expressions and `=` initializers.
///
/// Stops before `;` or before a token that cannot continue the list. The
/// semicolon is left for the caller to consume or diagnose as the enclosing
/// statement terminator.
#[must_use]
pub(crate) fn parse_variable_declarators(
&mut self,
) -> ArenaVec<'arena, VariableDeclaratorRef<'src, 'arena>> {
use VariableDeclaratorParseState::{ExpectingDeclarator, ExpectingSeparator};
declaration_keyword_position: TokenPosition,
variable_type_span: TokenSpan,
) -> ast::DeclaratorsList<'src, 'arena> {
use VariableDeclaratorListParseStep::{ExpectingDeclarator, ExpectingSeparator};
let mut declarators = self.arena.vec();
let mut parser_state = ExpectingDeclarator;
let mut state = VariableDeclaratorListParseState::new(
self,
declaration_keyword_position,
variable_type_span,
);
// Progress invariant: every branch that continues the loop consumes at
// least one token. Branches that only report an error must stop
// the loop, so the parser remains finite even if
// `ensure_forward_progress` is only a debug/safety check.
while let Some((next_token, next_token_position)) = self.peek_token_and_position() {
match (parser_state, next_token) {
match (state.parse_step, next_token) {
(ExpectingDeclarator, Token::Semicolon) => {
self.report_error_here(ParseErrorKind::DeclEmptyVariableDeclarations);
return declarators;
self.report_missing_variable_declarator_at_current_token(&state);
return state.declarators;
}
(ExpectingDeclarator, Token::Comma) => {
if self
.recover_empty_variable_declarator(next_token_position)
.recover_empty_variable_declarator_run(next_token_position, &state)
.is_break()
{
return declarators;
return state.declarators;
}
}
(ExpectingDeclarator, _) => {
(ExpectingDeclarator, _) if next_token.is_valid_identifier_name() => {
if self
.parse_variable_declarator_into(&mut declarators)
.parse_and_push_variable_declarator(&mut state)
.is_break()
{
// Breaking means we've failed to parse declarator
self.report_error_here(ParseErrorKind::DeclEmptyVariableDeclarations);
break;
}
parser_state = ExpectingSeparator;
}
// A token that cannot start a declarator ends the list.
// Report whether the whole list is missing or only the current
// declarator slot is missing, then leave the enclosing
// terminator for the caller to report.
(ExpectingDeclarator, _) => {
self.report_missing_variable_declarator_at_current_token(&state);
break;
}
(ExpectingSeparator, Token::Comma) => {
self.advance();
parser_state = ExpectingDeclarator;
self.advance(); // ','
state.set_declarator_expected(next_token_position);
}
(ExpectingSeparator, Token::Semicolon) => break,
(ExpectingSeparator, _) => {
// Missing-separator recovery is intentionally stricter than
// normal declarator parsing, so identifier-like keywords are
// not treated as continuations (no `is_valid_identifier_name`).
(ExpectingSeparator, Token::Identifier) => {
if self
.recover_missing_variable_declarator_separator(
.recover_missing_separator_before_variable_declarator(
next_token_position,
&mut declarators,
&mut state,
)
.is_break()
{
break;
}
}
// Any other invalid cases in place of separator should be
// handled as errors by the caller to avoid error cascade.
(ExpectingSeparator, _) => break,
}
// Guard against parser bugs that would otherwise leave
// block parsing stuck on the same token.
self.ensure_forward_progress(next_token_position);
}
// In case of reaching EOF here, it does not matter if we emit
// an additional diagnostic.
// The caller is expected to report the more relevant enclosing error.
declarators
// At end-of-file, the caller is expected to report the more relevant
// enclosing error, so this parser does not add another diagnostic.
state.declarators
}
fn recover_empty_variable_declarator(
/// Reports a missing variable declarator and attaches declaration context.
fn report_missing_variable_declarator_at_current_token(
&mut self,
state: &VariableDeclaratorListParseState<'src, 'arena>,
) {
let error_position = self.peek_position_or_eof();
self.make_error_at(state.missing_declarator_error_kind(), error_position)
.related_token("declaration_keyword", state.declaration_keyword_position)
.related("variable_type", state.variable_type_span)
.maybe_related_token("separator", state.pending_separator_position)
.report(self);
}
/// Recovers from one or more empty declarator slots produced by commas.
///
/// Called when the current token is `,`. Returns [`ControlFlow::Continue`]
/// only when the next remaining token can start a declarator.
fn recover_empty_variable_declarator_run(
&mut self,
error_start_position: TokenPosition,
state: &VariableDeclaratorListParseState<'src, 'arena>,
) -> ControlFlow<()> {
while self.peek_token() == Some(Token::Comma) {
let missing_first_declarator =
state.declarators.is_empty() && state.pending_separator_position.is_none();
while let Some(Token::Comma) = self.peek_token() {
self.advance();
}
self.make_error_at_last_consumed(ParseErrorKind::DeclEmptyVariableDeclarations)
let mut error = self
.make_error_at_last_consumed(ParseErrorKind::VariableDeclaratorExpected)
.widen_error_span_from(error_start_position)
.report_error(self);
if matches!(self.peek_token(), Some(Token::Semicolon) | None) {
ControlFlow::Break(())
} else {
.extend_blame_start_to_covered_start()
.related_token("declaration_keyword", state.declaration_keyword_position)
.related("variable_type", state.variable_type_span)
.maybe_related_token("separator", state.pending_separator_position)
.with_flag("empty_declarator_run");
if missing_first_declarator {
error = error.with_flag("first_declarator");
}
error.report_error(self);
if self
.peek_token()
.is_some_and(|token| token.is_valid_identifier_name())
{
ControlFlow::Continue(())
} else {
ControlFlow::Break(())
}
}
fn parse_variable_declarator_into(
/// Parses one variable declarator and appends it to `state`.
///
/// Returns [`ControlFlow::Continue`] only after consuming at least the
/// declarator name and advancing `state` to the separator phase. Returns
/// [`ControlFlow::Break`] on parse failure so the caller does not retry at
/// the same token.
fn parse_and_push_variable_declarator(
&mut self,
declarators: &mut ArenaVec<'arena, VariableDeclaratorRef<'src, 'arena>>,
state: &mut VariableDeclaratorListParseState<'src, 'arena>,
) -> ControlFlow<()> {
if let Some(parsed_declarator) = self
.parse_variable_declarator()
.parse_variable_declarator(state.declaration_keyword_position)
.sync_error_until(self, SyncLevel::StatementStart)
.ok_or_report(self)
{
declarators.push(parsed_declarator);
state.push_declarator(parsed_declarator);
ControlFlow::Continue(())
} else {
// Keep the fallback so changes in declarator recovery do not turn
// this helper into an unconditional continuation.
ControlFlow::Break(())
}
}
fn recover_missing_variable_declarator_separator(
/// Recovers from a declarator that starts without a preceding comma.
///
/// Returns [`ControlFlow::Continue`] only after consuming the declarator
/// and reporting a missing-separator diagnostic.
/// Returns [`ControlFlow::Break`] on parse failure so the caller does not
/// retry at the same token.
fn recover_missing_separator_before_variable_declarator(
&mut self,
error_start_position: TokenPosition,
declarators: &mut ArenaVec<'arena, VariableDeclaratorRef<'src, 'arena>>,
state: &mut VariableDeclaratorListParseState<'src, 'arena>,
) -> ControlFlow<()> {
if let Some(parsed_declarator) = self
.parse_variable_declarator()
.parse_variable_declarator(state.declaration_keyword_position)
.widen_error_span_from(error_start_position)
.sync_error_until(self, SyncLevel::StatementStart)
.ok_or_report(self)
{
self.make_error_at_last_consumed(ParseErrorKind::DeclNoSeparatorBetweenVariableDeclarations)
.widen_error_span_from(error_start_position)
.report_error(self);
declarators.push(parsed_declarator);
self.make_error_at_last_consumed(
ParseErrorKind::VariableDeclaratorListMissingSeparator,
)
.widen_error_span_from(error_start_position)
.related_token("declaration_keyword", state.declaration_keyword_position)
.maybe_related("last_declarator", state.previous_declarator_span)
.report_error(self);
state.push_declarator(parsed_declarator);
ControlFlow::Continue(())
} else {
// Keep the fallback so changes in declarator recovery do not turn
// this helper into an unconditional continuation.
ControlFlow::Break(())
}
}
/// Parses a single variable declarator.
fn parse_variable_declarator(
&mut self,
declaration_keyword_position: TokenPosition,
) -> ParseResult<'src, 'arena, VariableDeclaratorRef<'src, 'arena>> {
let name = self.parse_identifier(ParseErrorKind::DeclBadVariableIdentifier)?;
let array_size = self.parse_optional_array_size();
let initializer = self.parse_optional_variable_initializer();
let span = TokenSpan::range(name.0, self.last_consumed_position_or_start());
let variable_name = self.parse_identifier(ParseErrorKind::VariableDeclaratorExpected)?;
let array_size =
self.parse_optional_array_size(declaration_keyword_position, variable_name.0);
let initializer = self.parse_optional_variable_initializer(variable_name.0);
let declarator_span =
TokenSpan::range(variable_name.0, self.last_consumed_position_or_start());
Ok(self.arena.alloc_node(
VariableDeclarator {
name,
name: variable_name,
initializer,
array_size,
},
span,
declarator_span,
))
}
fn parse_optional_array_size(&mut self) -> OptionalExpression<'src, 'arena> {
/// Parses an optional array-size expression.
///
/// Returns [`None`] when no array size is present or when the size
/// expression cannot be recovered.
fn parse_optional_array_size(
&mut self,
declaration_keyword_position: TokenPosition,
variable_name_position: TokenPosition,
) -> OptionalExpression<'src, 'arena> {
if !self.eat(Token::LeftBracket) {
return None;
}
let array_size_expression = self.parse_expression();
let left_bracket_position = self.last_consumed_position_or_start();
let array_size_expression = self
.parse_required_expression_with_context(
ParseErrorKind::VariableDeclaratorArraySizeExpected,
variable_name_position,
left_bracket_position,
)
.sync_error_until(self, SyncLevel::CloseBracket)
.ok_or_report(self);
self.expect(
Token::RightBracket,
ParseErrorKind::DeclExpectedRightBracketAfterArraySize,
ParseErrorKind::VariableDeclaratorArraySizeMissingClosingBracket,
)
.sync_error_at(self, SyncLevel::CloseBracket)
.related_token("declaration_keyword", declaration_keyword_position)
.related_token("open_bracket", left_bracket_position)
.sync_error_at_matching_delimiter(self, left_bracket_position)
.report_error(self);
Some(array_size_expression)
array_size_expression
}
fn parse_optional_variable_initializer(&mut self) -> OptionalExpression<'src, 'arena> {
self.eat(Token::Assign).then(|| self.parse_expression())
/// Parses an optional initializer.
///
/// Returns [`None`] both when the initializer is absent and when its
/// expression cannot be recovered. This makes the recovered AST ambiguous,
/// but that is acceptable because callers should treat it only as a
/// best-effort result after a reported syntax error.
fn parse_optional_variable_initializer(
&mut self,
variable_name_position: TokenPosition,
) -> OptionalExpression<'src, 'arena> {
if self.eat(Token::Assign) {
let assignment_operator_position = self.last_consumed_position_or_start();
self.parse_required_expression_with_context(
ParseErrorKind::VariableDeclaratorInitializerExpected,
variable_name_position,
assignment_operator_position,
)
.sync_error_until(self, SyncLevel::ListSeparator)
.ok_or_report(self)
} else {
None
}
}
}

4
rottlib/src/parser/grammar/expression/control_flow/mod.rs
View File

@ -1,8 +1,8 @@
//! Control expression parsing for Fermented `UnrealScript`.
//! Control expression parsing for Fermented UnrealScript.
//!
//! ## Condition boundary recovery and legacy compatibility
//!
//! Fermented `UnrealScript` allows omitting parentheses `(...)` around
//! Fermented UnrealScript allows omitting parentheses `(...)` around
//! condition expressions of `if`/`while`/`until` and similar constructs.
//! Conditions are therefore parsed as ordinary expressions by default.
//!

102
rottlib/src/parser/grammar/expression/identifier.rs
View File

@ -1,35 +1,33 @@
//! Identifier parsing for Fermented `UnrealScript`.
//! Identifier parsing for Fermented UnrealScript.
//!
//! Provides shared routines for parsing both regular and qualified identifiers,
//! e.g. `KFChar.ZombieClot`.
//! Provides shared routines for parsing plain identifiers and dot-qualified
//! identifier paths, e.g. `KFChar.ZombieClot`.
use crate::arena::{self, ArenaVec};
use crate::ast::{IdentifierToken, QualifiedIdentifier, QualifiedIdentifierRef};
use crate::arena;
use crate::ast::{self, IdentifierToken, QualifiedIdentifierRef};
use crate::lexer::{self, Token, TokenSpan};
use crate::parser::{ParseErrorKind, ParseResult, Parser, ResultRecoveryExt};
use crate::parser::{ParseErrorKind, ParseResult, Parser, ResultRecoveryExt, diagnostic_labels};
impl<'src, 'arena> Parser<'src, 'arena> {
/// Parses an identifier.
///
/// On failure (unexpected end-of-file or a token that cannot be used as an
/// identifier), produces `invalid_identifier_error_kind`.
/// Consumes one identifier token and returns its position.
/// Produces `invalid_identifier_error_kind` if the next token is missing or
/// cannot be used as an identifier without consuming anything.
pub(crate) fn parse_identifier(
&mut self,
invalid_identifier_error_kind: ParseErrorKind,
) -> ParseResult<'src, 'arena, IdentifierToken> {
let (token, token_position) =
self.require_token_and_position(invalid_identifier_error_kind)?;
let identifier = Parser::identifier_token_from_token(token, token_position)
.ok_or_else(|| self.make_error_at_last_consumed(invalid_identifier_error_kind))?;
let identifier = Self::identifier_token_from_token(token, token_position)
.ok_or_else(|| self.make_error_at(invalid_identifier_error_kind, token_position))?;
self.advance();
Ok(identifier)
}
/// Returns an [`IdentifierToken`] for `token` if it is valid as an
/// identifier name.
///
/// This helper performs only token-to-identifier validation/wrapping;
/// it does not consume input from the parser.
/// Returns an [`IdentifierToken`] if `token` can be used as
/// an identifier name.
pub(crate) fn identifier_token_from_token(
token: Token,
token_position: lexer::TokenPosition,
@ -39,42 +37,74 @@ impl<'src, 'arena> Parser<'src, 'arena> {
.then_some(IdentifierToken(token_position))
}
/// Parses a qualified (dot-separated) identifier path,
/// e.g. `KFChar.ZombieClot`.
/// Parses a dot-qualified identifier path.
///
/// This is used for name paths where each segment must be
/// a valid identifier and segments are separated by `.` tokens.
/// Accepts both a single identifier, such as `KFChar`,
/// and a qualified path, such as `KFChar.ZombieClot`.
///
/// On failure produces an error of specified [`ParseErrorKind`]
/// `invalid_identifier_error_kind`.
/// Produces `invalid_identifier_error_kind` when the first identifier or a
/// segment after `.` is missing or invalid.
pub(crate) fn parse_qualified_identifier(
&mut self,
invalid_identifier_error_kind: ParseErrorKind,
) -> ParseResult<'src, 'arena, QualifiedIdentifierRef<'arena>> {
let head = self.parse_identifier(invalid_identifier_error_kind)?;
let mut tail = None;
let span_start = head.0;
let mut span_end = span_start;
let mut identifier_span = TokenSpan::new(head.0);
while let Some((Token::Period, dot_position)) = self.peek_token_and_position() {
self.advance(); // '.'
let next_segment = match self
.parse_identifier(invalid_identifier_error_kind)
.widen_error_span_from(head.0)
{
Ok(next_segment) => next_segment,
Err(error) => return Err(error.related_token("qualifier_dot", dot_position)),
let Some((segment_token, segment_position)) = self.peek_token_and_position() else {
let next_position = self.peek_position_or_eof();
return Err(self
.make_error_at(invalid_identifier_error_kind, next_position)
.widen_error_span_from(identifier_span.start)
.related_token(diagnostic_labels::QUALIFIED_IDENTIFIER_DOT, dot_position));
};
span_end = next_segment.0;
let tail_vec = tail.get_or_insert_with(|| ArenaVec::new_in(self.arena));
tail_vec.push(next_segment);
let Some(segment) = Self::identifier_token_from_token(segment_token, segment_position)
else {
let invalid_segment_position = segment_position;
let consumed_recovery_tokens = self.recover_invalid_qualified_identifier_tail();
let diagnostic_position = if consumed_recovery_tokens {
self.last_consumed_position_or_start()
} else {
invalid_segment_position
};
return Err(self
.make_error_at(invalid_identifier_error_kind, diagnostic_position)
.widen_error_span_from(identifier_span.start)
.blame_token(invalid_segment_position)
.related_token(diagnostic_labels::QUALIFIED_IDENTIFIER_DOT, dot_position));
};
self.advance(); // consume segment
identifier_span.extend_to(segment.0);
let remaining_segments_vec =
tail.get_or_insert_with(|| arena::ArenaVec::new_in(self.arena));
remaining_segments_vec.push(segment);
}
Ok(arena::ArenaNode::new_in(
QualifiedIdentifier { head, tail },
TokenSpan::range(span_start, span_end),
ast::QualifiedIdentifier { head, tail },
identifier_span,
self.arena,
))
}
/// Recovers from an invalid segment in a qualified identifier tail.
///
/// Recovery skips only additional `.segment` suffixes after
/// the invalid token. Returns whether any recovery token was consumed.
fn recover_invalid_qualified_identifier_tail(&mut self) -> bool {
let mut consumed_any = false;
while self.eat(Token::Period) {
consumed_any = true;
if self
.peek_token()
.is_some_and(|token| token.is_valid_identifier_name())
{
self.advance();
consumed_any = true;
}
}
consumed_any
}
}

26
rottlib/src/parser/grammar/expression/literals.rs
View File

@ -1,4 +1,4 @@
//! Literal decoding for Fermented `UnrealScript`.
//! Literal decoding for Fermented UnrealScript.
//!
//! This module defines the semantic rules for interpreting literal tokens
//! produced by the lexer. It is responsible only for *decoding* the textual
@ -7,6 +7,7 @@
//! The rules implemented here intentionally mirror the quirks of
//! Unreal Engine 2’s `UnrealScript`.
use crate::lexer::TokenPosition;
use crate::parser::{ParseErrorKind, ParseResult};
impl<'src, 'arena> crate::parser::Parser<'src, 'arena> {
@ -24,7 +25,11 @@ impl<'src, 'arena> crate::parser::Parser<'src, 'arena> {
///
/// On failure, returns [`ParseErrorKind::InvalidNumericLiteral`] at
/// the parser's current cursor position.
pub(crate) fn decode_integer_literal(&self, literal: &str) -> ParseResult<'src, 'arena, u128> {
pub(crate) fn decode_integer_literal(
&self,
literal: &str,
literal_position: TokenPosition,
) -> ParseResult<'src, 'arena, u128> {
let (base, content) = match literal.split_at_checked(2) {
Some(("0b" | "0B", stripped)) => (2, stripped),
Some(("0o" | "0O", stripped)) => (8, stripped),
@ -32,8 +37,9 @@ impl<'src, 'arena> crate::parser::Parser<'src, 'arena> {
_ => (10, literal),
};
let digits_without_underscores = content.replace('_', "");
u128::from_str_radix(&digits_without_underscores, base)
.map_err(|_| self.make_error_at_last_consumed(ParseErrorKind::InvalidNumericLiteral))
u128::from_str_radix(&digits_without_underscores, base).map_err(|_| {
self.make_error_at(ParseErrorKind::InvalidNumericLiteral, literal_position)
})
}
/// Decodes a float literal as `f64`, following the permissive and only
@ -66,7 +72,11 @@ impl<'src, 'arena> crate::parser::Parser<'src, 'arena> {
/// On failure, this function returns
/// [`ParseErrorKind::InvalidNumericLiteral`] at the current parser
/// position.
pub(crate) fn decode_float_literal(&self, literal: &str) -> ParseResult<'src, 'arena, f64> {
pub(crate) fn decode_float_literal(
&self,
literal: &str,
literal_position: TokenPosition,
) -> ParseResult<'src, 'arena, f64> {
let content = literal
.strip_suffix('f')
.or_else(|| literal.strip_suffix('F'))
@ -77,9 +87,9 @@ impl<'src, 'arena> crate::parser::Parser<'src, 'arena> {
.nth(1)
.and_then(|(period_index, _)| content.get(..period_index))
.unwrap_or(content);
content
.parse::<f64>()
.map_err(|_| self.make_error_at_last_consumed(ParseErrorKind::InvalidNumericLiteral))
content.parse::<f64>().map_err(|_| {
self.make_error_at(ParseErrorKind::InvalidNumericLiteral, literal_position)
})
}
/// Unescapes a tokenized string literal into an arena string.

2
rottlib/src/parser/grammar/expression/mod.rs
View File

@ -1,4 +1,4 @@
//! Expression parsing for Fermented `UnrealScript`.
//! Expression parsing for Fermented UnrealScript.
//!
//! This module group implements the language's expression parser around a
//! Pratt-style core. It is split into small submodules by role: precedence,

60
rottlib/src/parser/grammar/expression/pratt.rs
View File

@ -1,4 +1,4 @@
//! Core of the expression parser for Fermented `UnrealScript`.
//! Core of the expression parser for Fermented UnrealScript.
//!
//! This module implements a Pratt-style parser for the language's expression
//! grammar, supporting:
@ -13,7 +13,7 @@
//! operators bind. Infix parsing uses the pair of binding powers returned by
//! [`super::precedence::infix_precedence_ranks`] to encode associativity.
//! The parser infrastructure supports both left- and right-associative
//! operators, but Fermented `UnrealScript` currently defines only
//! operators, but Fermented UnrealScript currently defines only
//! left-associative ones.
//!
//! ## Postfix operator vs "selectors"
@ -35,7 +35,7 @@
use crate::ast::{self, Expression, ExpressionRef};
use crate::lexer::TokenPosition;
use crate::parser::{
self, ParseErrorKind, ParseExpressionResult, Parser, ResultRecoveryExt, diagnostic_labels,
ParseErrorKind, ParseExpressionResult, Parser, ResultRecoveryExt, SyncLevel, diagnostic_labels,
};
pub use super::precedence::PrecedenceRank;
@ -59,34 +59,30 @@ fn forbids_postfix_operators(expression: &ExpressionRef<'_, '_>) -> bool {
}
impl<'src, 'arena> Parser<'src, 'arena> {
// TODO: success here guaranees progress
/// Parses an expression.
///
/// Always returns some expression node; any syntax errors are reported
/// through the parser's diagnostics.
/// Returning non-erroneous node guarantees that parser has advanced
/// at least one token.
#[must_use]
pub fn parse_expression(&mut self) -> ExpressionRef<'src, 'arena> {
self.parse_expression_with_min_precedence_rank(PrecedenceRank::LOOSEST)
.sync_error_until(self, parser::SyncLevel::ExpressionStart)
.sync_error_until(self, SyncLevel::ExpressionStart)
.unwrap_or_fallback(self)
}
/// Parses an expression in a grammar position where an expression is
/// required.
///
/// This is the checked variant of [`Parser::parse_expression`]. If the next
/// token is known not to be a valid expression starter, this reports
/// `bad_start_error_kind`, consumes the bad token, and starts panic-mode
/// recovery until [`crate::parser::SyncLevel::ExpressionStart`].
/// If the next token cannot start an expression, this reports
/// `bad_start_error_kind` at that token, starts expression-start recovery
/// and returns an error expression result.
///
/// `required_by_position` identifies the token or construct that created
/// the requirement for an expression. It is attached to the diagnostic with
/// the [`diagnostic_labels::EXPRESSION_REQUIRED_BY`] label.
///
/// `expression_context_position` identifies the local syntactic anchor after
/// which the expression was expected. It is attached to the diagnostic with
/// the [`diagnostic_labels::EXPRESSION_EXPECTED_AFTER`] label.
pub(super) fn parse_required_expression(
pub(crate) fn parse_required_expression(
&mut self,
bad_start_error_kind: ParseErrorKind,
required_by_position: TokenPosition,
@ -96,7 +92,7 @@ impl<'src, 'arena> Parser<'src, 'arena> {
return Err(self
.make_error_at(bad_start_error_kind, error_position)
.sync_error_until(self, crate::parser::SyncLevel::ExpressionStart)
.sync_error_until(self, SyncLevel::ExpressionStart)
.blame_token(error_position)
.related_token(
diagnostic_labels::EXPRESSION_REQUIRED_BY,
@ -107,7 +103,22 @@ impl<'src, 'arena> Parser<'src, 'arena> {
self.parse_expression_with_min_precedence_rank(PrecedenceRank::LOOSEST)
}
pub(super) fn parse_required_expression_with_context(
/// Parses an expression in a grammar position where an expression is
/// required, and records the local syntactic context where it was expected.
///
/// If the next token cannot start an expression, this reports
/// `bad_start_error_kind` at that token, starts expression-start recovery,
/// and returns an error expression result.
///
/// `required_by_position` identifies the token or construct that created
/// the requirement for an expression. It is attached to the diagnostic with
/// the [`diagnostic_labels::EXPRESSION_REQUIRED_BY`] label.
///
/// `expression_context_position` identifies the local syntactic anchor
/// after which the expression was expected. It is attached to
/// the diagnostic with the [`diagnostic_labels::EXPRESSION_EXPECTED_AFTER`]
/// label.
pub(crate) fn parse_required_expression_with_context(
&mut self,
bad_start_error_kind: ParseErrorKind,
required_by_position: TokenPosition,
@ -118,7 +129,7 @@ impl<'src, 'arena> Parser<'src, 'arena> {
return Err(self
.make_error_at(bad_start_error_kind, error_position)
.sync_error_until(self, crate::parser::SyncLevel::ExpressionStart)
.sync_error_until(self, SyncLevel::ExpressionStart)
.blame_token(error_position)
.related_token(
diagnostic_labels::EXPRESSION_REQUIRED_BY,
@ -133,6 +144,11 @@ impl<'src, 'arena> Parser<'src, 'arena> {
self.parse_expression_with_min_precedence_rank(PrecedenceRank::LOOSEST)
}
/// Creates an error expression at `position`.
///
/// The returned node has a single-token span anchored at `position` and
/// can be used as a fallback expression when parsing cannot produce
/// a valid one.
pub(super) fn make_error_expression_at(
&self,
position: TokenPosition,
@ -149,7 +165,7 @@ impl<'src, 'arena> Parser<'src, 'arena> {
fn parse_expression_with_min_precedence_rank(
&mut self,
min_precedence_rank: PrecedenceRank,
) -> parser::ParseExpressionResult<'src, 'arena> {
) -> ParseExpressionResult<'src, 'arena> {
let mut left_hand_side = self.parse_prefix_or_primary()?;
left_hand_side = self.parse_selectors_after(left_hand_side)?;
// We disallow only postfix operators after expression forms that
@ -174,7 +190,7 @@ impl<'src, 'arena> Parser<'src, 'arena> {
/// Parses a prefix or primary expression (Pratt parser's "nud" or
/// null denotation).
fn parse_prefix_or_primary(&mut self) -> parser::ParseExpressionResult<'src, 'arena> {
fn parse_prefix_or_primary(&mut self) -> ParseExpressionResult<'src, 'arena> {
let (token, token_lexeme, token_position) =
self.require_token_lexeme_and_position(ParseErrorKind::ExpressionExpected)?;
// Avoid advancing over an obviously wrong token;
@ -226,7 +242,7 @@ impl<'src, 'arena> Parser<'src, 'arena> {
&mut self,
mut left_hand_side: ExpressionRef<'src, 'arena>,
min_precedence_rank: PrecedenceRank,
) -> parser::ParseExpressionResult<'src, 'arena> {
) -> ParseExpressionResult<'src, 'arena> {
while let Some((operator, right_precedence_rank)) =
self.peek_infix_with_min_precedence_rank(min_precedence_rank)
{
@ -249,9 +265,7 @@ impl<'src, 'arena> Parser<'src, 'arena> {
&mut self,
) -> Option<(ast::PostfixOperator, crate::lexer::TokenPosition)> {
let (token, token_position) = self.peek_token_and_position()?;
let Ok(operator) = ast::PostfixOperator::try_from(token) else {
return None;
};
let operator = ast::PostfixOperator::try_from(token).ok()?;
Some((operator, token_position))
}

2
rottlib/src/parser/grammar/expression/precedence.rs
View File

@ -1,4 +1,4 @@
//! Precedence tables for Fermented `UnrealScript` operators.
//! Precedence tables for Fermented UnrealScript operators.
//!
//! These values don't follow the usual *binding power* convention for
//! a Pratt parser, where tighter binding corresponds to a larger number.\

132
rottlib/src/parser/grammar/expression/primary/mod.rs
View File

@ -54,12 +54,20 @@
//! `(`, `switch` is parsed as a `switch` expression; otherwise it remains
//! available as an identifier.
use crate::ast::{Expression, ExpressionRef, OptionalExpression};
use crate::ast::{Expression, ExpressionRef, OptionalExpression, QualifiedIdentifierRef};
use crate::lexer::{Keyword, Token, TokenPosition, TokenSpan};
use crate::parser::{ParseErrorKind, ParseExpressionResult, Parser, ResultRecoveryExt, SyncLevel};
use crate::parser::{
ParseError, ParseErrorKind, ParseExpressionResult, ParseResult, Parser, ResultRecoveryExt,
SyncLevel,
};
mod new;
pub(crate) struct ParsedClassTypeArgument<'arena> {
pub(crate) type_name: QualifiedIdentifierRef<'arena>,
pub(crate) right_angle_bracket_position: TokenPosition,
}
impl<'src, 'arena> Parser<'src, 'arena> {
/// Parses a primary expression starting from the provided token.
///
@ -84,12 +92,12 @@ impl<'src, 'arena> Parser<'src, 'arena> {
) -> ParseExpressionResult<'src, 'arena> {
Ok(match token {
Token::IntegerLiteral => {
let value = self.decode_integer_literal(token_lexeme)?;
let value = self.decode_integer_literal(token_lexeme, token_position)?;
self.arena
.alloc_node_at(Expression::Integer(value), token_position)
}
Token::FloatLiteral => {
let value = self.decode_float_literal(token_lexeme)?;
let value = self.decode_float_literal(token_lexeme, token_position)?;
self.arena
.alloc_node_at(Expression::Float(value), token_position)
}
@ -147,7 +155,8 @@ impl<'src, 'arena> Parser<'src, 'arena> {
// These keywords remain valid identifiers unless the following
// tokens commit to the keyword-led form.
Keyword::For
if let Some(left_parenthesis_position) = self.peek_for_loop_header_left_parenthesis_position() =>
if let Some(left_parenthesis_position) =
self.peek_for_loop_header_left_parenthesis_position() =>
{
self.advance(); // `(`
self.parse_for_tail(token_position, left_parenthesis_position)
@ -247,77 +256,98 @@ impl<'src, 'arena> Parser<'src, 'arena> {
/// Parses a class type expression of the form `class<...>`.
///
/// Assumes the `class` keyword and following '<' token have already been
/// Assumes the `class` keyword and following `<` token have already been
/// consumed. Reports and recovers from malformed type syntax locally.
fn parse_class_type_tail(
&mut self,
class_keyword_position: TokenPosition,
left_angle_bracket_position: TokenPosition,
) -> ExpressionRef<'src, 'arena> {
match self.peek_token_and_position() {
Some((Token::Greater, right_angle_bracket_position)) => self
.report_missing_class_type_argument(
class_keyword_position,
left_angle_bracket_position,
right_angle_bracket_position,
),
Some((first_token, _)) if first_token.is_valid_identifier_name() => self
.parse_nonempty_class_type_tail(
class_keyword_position,
left_angle_bracket_position,
),
Some((_, bad_position)) => self.report_invalid_class_type_start(
class_keyword_position,
left_angle_bracket_position,
bad_position,
),
None => self.report_invalid_class_type_start(
class_keyword_position,
left_angle_bracket_position,
self.file.eof(),
),
}
let parsed = match self
.parse_class_type_argument_tail(class_keyword_position, left_angle_bracket_position)
{
Ok(parsed) => parsed,
Err(error) => return self.report_error_with_fallback(error),
};
self.arena.alloc_node_between(
Expression::ClassType(parsed.type_name),
class_keyword_position,
parsed.right_angle_bracket_position,
)
}
fn parse_nonempty_class_type_tail(
/// Parses a class type argument of the form `...>`.
///
/// Assumes the `class` keyword and following `<` token have already been
/// consumed. Returns the parsed type argument and closing `>` position.
pub(crate) fn parse_class_type_argument_tail(
&mut self,
class_keyword_position: TokenPosition,
left_angle_bracket_position: TokenPosition,
) -> ExpressionRef<'src, 'arena> {
let class_type = match self
) -> ParseResult<'src, 'arena, ParsedClassTypeArgument<'arena>> {
match self.peek_token_and_position() {
Some((Token::Greater, right_angle_bracket_position)) => {
self.advance();
Err(self.make_missing_class_type_argument_error(
class_keyword_position,
left_angle_bracket_position,
right_angle_bracket_position,
))
}
Some((first_token, _)) if first_token.is_valid_identifier_name() => self
.parse_nonempty_class_type_argument_tail(
class_keyword_position,
left_angle_bracket_position,
),
Some((_, bad_position)) => Err(self.make_invalid_class_type_start_error(
class_keyword_position,
left_angle_bracket_position,
bad_position,
)),
None => Err(self.make_invalid_class_type_start_error(
class_keyword_position,
left_angle_bracket_position,
self.file.eof(),
)),
}
}
/// Parses a nonempty class type argument followed by its closing `>`.
///
/// Assumes the next token can start a qualified type name.
fn parse_nonempty_class_type_argument_tail(
&mut self,
class_keyword_position: TokenPosition,
left_angle_bracket_position: TokenPosition,
) -> ParseResult<'src, 'arena, ParsedClassTypeArgument<'arena>> {
let type_name = self
.parse_qualified_identifier(ParseErrorKind::ClassTypeExpectedQualifiedTypeName)
.widen_error_span_from(class_keyword_position)
.extend_blame_to_next_token(self)
.sync_error_at(self, SyncLevel::CloseAngleBracket)
.related_token("class_keyword", class_keyword_position)
{
Ok(class_type) => class_type,
Err(error) => return self.report_error_with_fallback(error),
};
.related_token("class_keyword", class_keyword_position)?;
let right_angle_bracket_position = self
.expect(
Token::Greater,
ParseErrorKind::ClassTypeMissingClosingAngleBracket,
)
.widen_error_span_from(class_keyword_position)
.sync_error_at(self, SyncLevel::CloseAngleBracket)
.related_token("left_angle_bracket", left_angle_bracket_position)
.related_token("class_keyword", class_keyword_position)
.unwrap_or_fallback(self);
self.arena.alloc_node_between(
Expression::ClassType(class_type),
class_keyword_position,
.related_token("class_keyword", class_keyword_position)?;
Ok(ParsedClassTypeArgument {
type_name,
right_angle_bracket_position,
)
})
}
fn report_missing_class_type_argument(
fn make_missing_class_type_argument_error(
&mut self,
class_keyword_position: TokenPosition,
left_angle_bracket_position: TokenPosition,
right_angle_bracket_position: TokenPosition,
) -> ExpressionRef<'src, 'arena> {
self.advance();
) -> ParseError {
self.make_error_at_last_consumed(ParseErrorKind::ClassTypeMissingTypeArgument)
.widen_error_span_from(class_keyword_position)
.blame(TokenSpan::range(
@ -326,22 +356,20 @@ impl<'src, 'arena> Parser<'src, 'arena> {
))
.related_token("left_angle_bracket", left_angle_bracket_position)
.related_token("class_keyword", class_keyword_position)
.fallback(self)
}
fn report_invalid_class_type_start(
fn make_invalid_class_type_start_error(
&mut self,
class_keyword_position: TokenPosition,
left_angle_bracket_position: TokenPosition,
bad_position: TokenPosition,
) -> ExpressionRef<'src, 'arena> {
) -> ParseError {
self.make_error_at_last_consumed(ParseErrorKind::ClassTypeInvalidStart)
.widen_error_span_from(class_keyword_position)
.sync_error_at(self, SyncLevel::CloseAngleBracket)
.blame_token(bad_position)
.related_token("left_angle_bracket", left_angle_bracket_position)
.related_token("class_keyword", class_keyword_position)
.fallback(self)
}
/// Returns `true` iff the next token is definitely not a valid start of an

6
rottlib/src/parser/grammar/expression/selectors.rs
View File

@ -45,7 +45,11 @@ pub(super) enum ParsedArgumentSlot<'src, 'arena> {
impl<'src, 'arena> Parser<'src, 'arena> {
/// Parses zero or more postfix selectors after `left_hand_side`.
///
/// Returns the resulting expression after all contiguous selectors.
/// If the next token is not a selector starter, consumes nothing and
/// returns `left_hand_side` unchanged.
///
/// Each accepted selector consumes at least its leading token (`.`, `[` or
/// `(`) before selector-specific parsing or recovery is attempted.
pub(super) fn parse_selectors_after(
&mut self,
mut left_hand_side: ExpressionRef<'src, 'arena>,

8
rottlib/src/parser/grammar/function/definition.rs
View File

@ -1,4 +1,4 @@
//! Parsing of callable definitions for Fermented `UnrealScript`
//! Parsing of callable definitions for Fermented UnrealScript
//! (functions, events, delegates, operators).
use crate::arena::ArenaVec;
@ -239,10 +239,10 @@ impl<'src, 'arena> Parser<'src, 'arena> {
return None;
}
let value = match self.peek_token_and_lexeme() {
Some((Token::IntegerLiteral, lex)) => {
let value = match self.peek_token_lexeme_and_position() {
Some((Token::IntegerLiteral, lex, token_position)) => {
self.advance();
self.decode_integer_literal(lex).ok_or_report(self)
self.decode_integer_literal(lex, token_position).ok_or_report(self)
}
Some(_) => {
self.report_error_here(ParseErrorKind::OperatorPrecedenceNotIntegerLiteral);

2
rottlib/src/parser/grammar/function/lookahead.rs
View File

@ -1,4 +1,4 @@
//! Lookahead for callable headers in Fermented `UnrealScript`.
//! Lookahead for callable headers in Fermented UnrealScript.
use crate::lexer::{Keyword, Token};
use crate::parser::Parser;

76
rottlib/src/parser/grammar/statement.rs
View File

@ -1,7 +1,7 @@
//! Statement parsing for the language front-end.
//!
//! Implements a simple recursive-descent parser for
//! *Fermented `UnrealScript` statements*.
//! *Fermented UnrealScript statements*.
use crate::ast::{Statement, StatementRef};
use crate::lexer::{Keyword, Token, TokenSpan};
@ -13,14 +13,13 @@ impl<'src, 'arena> crate::parser::Parser<'src, 'arena> {
/// Does not consume a trailing `;` except for [`Statement::Empty`].
/// The caller handles semicolons (WRONG NOW - WE MUST HANDLE THEM). Returns [`Some`] if a statement is
/// recognized; otherwise [`None`].
/// ALSO WE SPECIFICALLY DONT HANDLE EXPRESSION TYPE STATEMENTS
/// ALSO WE SPECIFICALLY DONT HANDLE EXPRESSION TYPE STATEMENTS - on them also return NONE!!!!
/// Basically only generates errors if statement is recognized, but got fucked up.
#[must_use]
pub(crate) fn parse_statement(&mut self) -> Option<StatementRef<'src, 'arena>> {
let Some((token, lexeme, position)) = self.peek_token_lexeme_and_position() else {
self.report_error_here(ParseErrorKind::UnexpectedEndOfFile);
return None;
};
match token {
// Empty statement
Token::Semicolon => {
@ -30,17 +29,41 @@ impl<'src, 'arena> crate::parser::Parser<'src, 'arena> {
.alloc_node(Statement::Empty, TokenSpan::new(position)),
)
}
// UnrealScript `local` declaration
Token::Keyword(Keyword::Local) => {
self.advance(); // `local`
let start = position;
let type_spec = self.parse_type_specifier().unwrap_or_fallback(self);
let declarators = self.parse_variable_declarators();
// TODO: parse
let span = TokenSpan::range(start, self.last_consumed_position_or_start());
let Some(type_spec) = self
.parse_type_specifier()
.sync_error_at(self, crate::parser::SyncLevel::StatementTerminator)
.ok_or_report(self)
else {
return Some(crate::arena::ArenaNode::new_in(
Statement::Error,
crate::lexer::TokenSpan::new(position),
self.arena,
));
};
let declarators = self.parse_variable_declarators(position, *type_spec.span());
let mut span = TokenSpan::range(start, self.last_consumed_position_or_start());
let mut error = if self.peek_token().is_some() {
self
.expect(
Token::Semicolon,
ParseErrorKind::LocalVariableDeclarationMissingSemicolon,
)
.widen_error_span_from(position)
.related_token("local_keyword", position)
} else {
Ok(self.peek_position_or_eof())
};
if let Some(last_declarator) = declarators.last() {
error = error.related("last_declarator", *last_declarator.span())
}
let semicolon_position = error.ok_or_report(self);
if let Some(semicolon_position) = semicolon_position {
span.extend_to(semicolon_position);
}
Some(self.arena.alloc_node(
Statement::LocalVariableDeclaration {
type_spec,
@ -49,8 +72,6 @@ impl<'src, 'arena> crate::parser::Parser<'src, 'arena> {
span,
))
}
// Label: Ident ':' (also tolerate Begin:/End:)
Token::Identifier | Token::Keyword(Keyword::Begin | Keyword::End)
if matches!(self.peek_token_at(1), Some(Token::Colon)) =>
{
@ -61,26 +82,23 @@ impl<'src, 'arena> crate::parser::Parser<'src, 'arena> {
TokenSpan::range(position, self.last_consumed_position_or_start()),
))
}
// Nested function/event/operator inside blocks
t if t == Token::Keyword(Keyword::Function)
|| t == Token::Keyword(Keyword::Event)
|| t.is_valid_function_modifier() =>
{
let f = self.parse_callable_definition();
let span = *f.span();
Some(self.arena.alloc_node(Statement::Function(f), span))
_ if is_function_starter(token) => {
let callable_definition = self.parse_callable_definition();
let span = *callable_definition.span();
Some(
self.arena
.alloc_node(Statement::Function(callable_definition), span),
)
}
// C-like variable declaration starting with a TypeSpec
/*token if self.looks_like_variable_declaration_start(token) => Some(
self.parse_variable_declaration_start()
.sync_error_until(self, SyncLevel::Statement)
.unwrap_or_fallback(self),
),*/
// Not a statement
_ => None,
}
}
}
fn is_function_starter(token: Token) -> bool {
token.is_valid_function_modifier()
|| token == Token::Keyword(Keyword::Function)
|| token == Token::Keyword(Keyword::Event)
}

3
rottlib/src/parser/mod.rs
View File

@ -1,4 +1,4 @@
//! Parser for Fermented `UnrealScript` (`FerUS`).
//! Parser for Fermented UnrealScript (`FerUS`).
//!
//! Consumes tokens from [`crate::lexer::TokenizedFile`] and allocates AST
//! nodes in [`crate::arena::Arena`]. Basic expressions use a Pratt parser;
@ -46,6 +46,7 @@ pub type ParseExpressionResult<'src, 'arena> =
pub(crate) mod diagnostic_labels {
pub(crate) const EXPRESSION_REQUIRED_BY: &str = "expression_required_by";
pub(crate) const EXPRESSION_EXPECTED_AFTER: &str = "expression_expected_after";
pub(crate) const QUALIFIED_IDENTIFIER_DOT: &str = "qualifier_dot";
}
// TODO: add some kind of bailing for infinite loops

32
rottlib/src/parser/recovery.rs
View File

@ -49,7 +49,7 @@ pub enum SyncLevel {
/// A statement boundary or statement starter.
///
/// Includes `;` and keywords that begin standalone statements /
/// Includes keywords that begin standalone statements /
/// statement-like control-flow forms.
StatementStart,
@ -333,6 +333,7 @@ pub trait ResultRecoveryExt<'src, 'arena, T>: Sized {
fn blame(self, blame_span: TokenSpan) -> Self;
fn related(self, tag: impl Into<String>, related_span: TokenSpan) -> Self;
fn with_flag(self, tag: impl Into<String>) -> Self;
fn blame_token(self, blame_position: TokenPosition) -> Self {
self.blame(TokenSpan::new(blame_position))
@ -349,6 +350,26 @@ pub trait ResultRecoveryExt<'src, 'arena, T>: Sized {
self.related(tag, TokenSpan::new(related_position))
}
fn maybe_related(self, tag: impl Into<String>, related_span: Option<TokenSpan>) -> Self {
if let Some(related_span) = related_span {
self.related(tag, related_span)
} else {
self
}
}
fn maybe_related_token(
self,
tag: impl Into<String>,
related_position: Option<TokenPosition>,
) -> Self {
if let Some(related_position) = related_position {
self.related(tag, TokenSpan::new(related_position))
} else {
self
}
}
/// Extends the right end of the error span up to but not including
/// the next token of the given sync `level`.
///
@ -437,6 +458,10 @@ impl<'src, 'arena, T> ResultRecoveryExt<'src, 'arena, T> for ParseResult<'src, '
self.map_err(|error| error.related(tag, related_span))
}
fn with_flag(self, tag: impl Into<String>) -> Self {
self.map_err(|error| error.with_flag(tag))
}
fn sync_error_until(mut self, parser: &mut Parser<'src, 'arena>, level: SyncLevel) -> Self {
if let Err(ref mut error) = self {
parser.recover_until(level);
@ -555,6 +580,11 @@ impl<'src, 'arena> ResultRecoveryExt<'src, 'arena, ()> for ParseError {
self
}
fn with_flag(mut self, tag: impl Into<String>) -> Self {
self.flags.insert(tag.into());
self
}
fn sync_error_until(mut self, parser: &mut Parser<'src, 'arena>, level: SyncLevel) -> Self {
parser.recover_until(level);
self.covered_span.end = std::cmp::max(

1031
rottlib/tests/parser_diagnostics/declarations.rs Normal file
View File

File diff suppressed because it is too large Load Diff

1
rottlib/tests/parser_diagnostics/mod.rs
View File

@ -7,6 +7,7 @@ use rottlib::parser::Parser;
mod block_items;
mod control_flow_expressions;
mod declarations;
mod primary_expressions;
mod selector_expressions;
mod switch_expressions;