//! # Tokenizer
//!
//! Converts raw source text into a lossless, position-aware stream of lexical
//! [`Token`]s, grouped *per physical line*, and returns it as
//! a [`TokenizedFile`]. A trailing newline terminates the last physical line
//! rather than introducing an additional empty line.
//!
//! Design goals:
//!
//! 1. **Lossless**: preserving complete information for each token, enough to
//!     recreate the original bytes without loss.
//! 2. **LSP readiness**: the LSP heavily relies on UTF-16 support, so we
//!     precompute lengths of each token in that encoding, making interfacing
//!     easier.
//!
//! ## Opt-in debug helpers
//!
//! Extra diagnostics become available in **debug builds** or when the crate is
//! compiled with `debug` feature enabled. They live in the [`debug_tools`]
//! extension trait, implemented for [`TokenizedFile`].

mod queries;
mod raw_lexer;
#[cfg(test)]
mod tests;
mod token;

use std::collections::HashMap;
use std::ops::Range;

use logos::Logos;

use crate::diagnostics::{Diagnostic, DiagnosticBuilder};
use raw_lexer::RawToken;

pub use raw_lexer::BraceKind;
pub use token::Keyword;
pub use token::Token;

/// Empirically chosen starting size for token buffer (used during tokenization)
/// that provides good performance.
const DEFAULT_TOKEN_BUFFER_CAPACITY: usize = 20_000;

// TODO: check this!!!
/// Visible fragment of a token on one physical line.
///
/// `columns` is an end-exclusive range inside the string returned by
/// [`TokenizedFile::line_text`] for that line.
#[derive(Clone, Debug, Hash, PartialEq, Eq)]
pub struct VisibleLineSpan {
    pub line: usize,
    pub columns: std::ops::Range<usize>,
}

/// A token together with its source text and precomputed UTF-16 length.
///
/// It does not store an absolute file position.
#[derive(Clone, Copy, Debug, Hash, PartialEq, Eq)]
pub struct TokenData<'src> {
    /// Kind of token that was lexed.
    pub token: Token,
    /// Underlying text that was lexed as the corresponding token.
    pub lexeme: &'src str,
    /// Length of the token in UTF-16 code units for the needs of easy seeking
    /// using given LSP cursor coordinates (line + UTF-16 offset).
    /// Precomputed for convenience.
    pub utf16_length: usize,
}

/// 0-based index of a token within the file-wide token buffer.
#[derive(Clone, Copy, Debug, Hash, PartialEq, Eq, PartialOrd, Ord, Default)]
pub struct TokenPosition(pub usize);

// All inclusive!
#[derive(Debug, Hash, PartialEq, Eq, PartialOrd, Ord, Clone, Copy)]
pub struct TokenSpan {
    pub start: TokenPosition,
    pub end: TokenPosition,
}

impl TokenSpan {
    // -------- existing coord-based API (unchanged externally) --------

    #[must_use]
    pub const fn merge(left_span: &Self, right_span: &Self) -> Self {
        Self {
            // assumes both were constructed in the same style; good enough for the refactor
            start: left_span.start,
            end: right_span.end,
        }
    }

    // -------- NEW: 4 constructors based on TokenIndex --------

    /// Single-token span from an index (coords are dummy for now).
    #[inline]
    #[must_use]
    pub const fn new(single_index: TokenPosition) -> Self {
        Self {
            start: single_index,
            end: single_index,
        }
    }

    /// Span from two indices (coords are dummy for now).
    #[inline]
    #[must_use]
    pub const fn range(from: TokenPosition, to: TokenPosition) -> Self {
        Self {
            start: from,
            end: to,
        }
    }

    /// Immutable extension by index (keeps coords as-is).
    #[inline]
    #[must_use]
    pub fn extended(&self, right_most_index: TokenPosition) -> Self {
        Self {
            start: self.start,
            end: std::cmp::max(self.end, right_most_index),
        }
    }

    /// In-place extension by index (coords unchanged).
    #[inline]
    pub fn extend_to(&mut self, right_most_index: TokenPosition) {
        if right_most_index > self.end {
            self.end = right_most_index;
        }
    }
}

/// A tokenized, lossless representation of an `UnrealScript` source file.
#[derive(Clone, Debug, PartialEq, Eq)]
pub struct TokenizedFile<'src> {
    /// Arena of every token span in this file.
    buffer: Vec<TokenData<'src>>,
    /// Mapping from physical line number to the tokens that belong to it.
    lines: Vec<Line>,
    /// Mapping token index to ranges of bytes that correspond to
    /// visible characters (i.e. all non line terminators) in its lines.
    ///
    /// Records only exists for multiline tokens and ranges can be empty for
    /// lines that only contain line break boundary.
    multi_line_map: HashMap<BufferIndex, Vec<VisibleByteRange>>,
    /// Sparse map between matched opening and closing delimiters.
    delimiter_matches: DelimiterMatches,
    /// Diagnostics produced during tokenization and delimiter matching.
    diagnostics: Vec<Diagnostic>,
    /// Simple flag for marking erroneous state.
    had_errors: bool,
}

/// An immutable iterator over all tokens in a [`TokenizedFile`], preserving
/// their order of appearance in the original source file.
///
/// After exhaustion it keeps returning [`None`].
#[must_use]
#[derive(Clone, Debug)]
pub struct Tokens<'file, 'src> {
    /// Position of the next token to be returned in the canonical file-wide
    /// token arena.
    cursor: TokenPosition,
    /// [`TokenizedFile`] whose tokens we're iterating over.
    source_file: &'file TokenizedFile<'src>,
}

/// Type for referring to line numbers.
type LineNumber = usize;

/// Type for specific tokens inside each [`Line`].
type BufferIndex = usize;

/// Type for describing sub-range of visible characters of a single line for
/// some token.
type VisibleByteRange = Range<usize>;

/// Representation of a single physical line of the source file.
///
/// Uses ranges instead of slices to avoid a self-referential relationship
/// with [`TokenizedFile`], which Rust forbids.
#[derive(Clone, Debug, Hash, PartialEq, Eq)]
struct Line {
    /// Token that began on an earlier line (`None` for standalone lines).
    continued_from: Option<LineNumber>,
    /// Contiguous tokens that started on this line (`start >= end` iff empty).
    local_range: Range<BufferIndex>,
}

/// Mutable state used while tokenizing a source file.
#[derive(Debug)]
struct Tokenizer<'src> {
    /// Arena that owns every [`TokenData`] produced for the file.
    buffer: Vec<TokenData<'src>>,
    /// Mapping from physical line number to the tokens that belong to it.
    lines: Vec<Line>,
    /// Mapping token index to ranges of bytes that correspond to
    /// visible characters in its lines.
    multi_line_map: HashMap<BufferIndex, Vec<VisibleByteRange>>,
    /// The 0-based physical line number that is currently being scanned.
    line_number: usize,
    /// Points to the first token (index in [`Tokenizer::buffer`]) not yet
    /// committed to `lines`, e.g. where the current *line* starts.
    uncommitted_start_index: usize,
    /// When a multi-line token is being scanned, stores the 0-based line
    /// on which it started; [`None`] otherwise.
    ///
    /// `Some(line_number)` iff the current line is within a multi-line token
    /// that started on `line_number`; it is consumed exactly once by
    /// [`Self::commit_current_line`].
    multi_line_start_line: Option<LineNumber>,
    /// Tracks delimiter pairs and delimiter diagnostics.
    delimiter_matcher: DelimiterMatcher,
    /// Tracks crude line indentation for delimiter recovery diagnostics.
    indent_tracker: IndentTracker,
    /// Diagnostics produced while tokenizing.
    diagnostics: Vec<Diagnostic>,
    /// Set to `true` if the lexer reported any error tokens.
    had_errors: bool,
}

impl<'src> TokenizedFile<'src> {
    /// Tokenizes `source` and returns a fresh [`TokenizedFile`].
    ///
    /// Its output is lossless and groups resulting tokens by physical lines.
    /// Error spans are preserved as [`Token::Error`].
    #[must_use]
    pub fn tokenize(source: &'src str) -> Self {
        let mut tokenizer = Tokenizer::new();
        let mut lexer = RawToken::lexer(source);

        while let Some(token_result) = lexer.next() {
            // Add `Token::Error` manually, since Logos won't do it for us.
            let token = token_result.unwrap_or_else(|()| {
                tokenizer.had_errors = true;
                RawToken::Error
            });
            let token_piece = make_token_data(Token::from(token), lexer.slice());
            let position = tokenizer.process_token_piece(token_piece);
            tokenizer.after_token(position);
        }
        tokenizer.into_tokenized_file()
    }

    /// Returns `true` if tokenization produced any error tokens.
    #[must_use]
    pub const fn has_errors(&self) -> bool {
        self.had_errors
    }

    /// Returns an iterator over all contained tokens in the order they appear
    /// in the original source file.
    ///
    /// Returns pairs of position and token data: `(TokenPosition, TokenData)`.
    pub const fn iter(&self) -> Tokens<'_, 'src> {
        Tokens::new(self)
    }

    /// Returns diagnostics produced during tokenization and delimiter matching.
    #[must_use]
    pub fn diagnostics(&self) -> &[Diagnostic] {
        &self.diagnostics
    }

    /// Returns the matching delimiter token, if this token is a matched delimiter.
    #[must_use]
    pub fn matching_delimiter(&self, position: TokenPosition) -> Option<TokenPosition> {
        self.delimiter_matches.mate_of(position)
    }
}

impl Line {
    /// Creates a standalone line that owns a contiguous slice in
    /// the [`TokenizedFile::buffer`] arena.
    const fn standalone(locals: Range<BufferIndex>) -> Self {
        Self {
            continued_from: None,
            local_range: locals,
        }
    }

    /// Creates a line that is part of a multi-line token started on
    /// another line, referencing the 0-based index of its origin.
    const fn continued(carried: LineNumber) -> Self {
        Self {
            continued_from: Some(carried),
            local_range: 0..0,
        }
    }

    /// Creates a line that is part of a multi-line token started on
    /// another line and also contains additional tokens local to itself.
    const fn continued_with_tokens(carried: LineNumber, locals: Range<BufferIndex>) -> Self {
        Self {
            continued_from: Some(carried),
            local_range: locals,
        }
    }

    /// Returns a range of tokens inside [`TokenizedFile::buffer`] that start
    /// on this line.
    ///
    /// [`None`] means there are no such tokens. Otherwise range is guaranteed
    /// to not be empty.
    fn local_range(&self) -> Option<Range<BufferIndex>> {
        if self.local_range.is_empty() {
            None
        } else {
            Some(self.local_range.clone())
        }
    }
}

impl<'src> Tokenizer<'src> {
    /// Returns an empty tokenizer state.
    fn new() -> Self {
        Self {
            buffer: Vec::with_capacity(DEFAULT_TOKEN_BUFFER_CAPACITY),
            lines: Vec::new(),
            multi_line_map: HashMap::new(),
            line_number: 0,
            uncommitted_start_index: 0,
            multi_line_start_line: None,
            delimiter_matcher: DelimiterMatcher::default(),
            indent_tracker: IndentTracker::new(),
            diagnostics: Vec::new(),
            had_errors: false,
        }
    }

    /// Handles a token span and dispatches to the appropriate handler.
    fn process_token_piece(&mut self, token_piece: TokenData<'src>) -> TokenPosition {
        let position = TokenPosition(self.buffer.len());
        if token_piece.token.can_span_lines() {
            self.process_multi_line_token(token_piece);
        } else {
            self.process_single_line_token(token_piece);
        }
        position
    }

    /// Handles simple tokens that *never* span multiple lines, allowing us to
    /// skip a lot of work.
    fn process_single_line_token(&mut self, token_piece: TokenData<'src>) {
        if token_piece.token.is_newline() {
            self.line_number += 1;
            self.buffer.push(token_piece);
            self.commit_current_line();
        } else {
            self.buffer.push(token_piece);
        }
    }

    /// Handles tokens that might contain one or more newline characters.
    fn process_multi_line_token(&mut self, token_piece: TokenData<'src>) {
        let start_line = self.line_number;
        let line_break_map = split_visible_line_segments(token_piece.lexeme);
        let newline_count = line_break_map.len().saturating_sub(1);

        // Needed for unterminated multi-line error tokens that reach EOF right
        // after a line break.
        let ends_with_newline =
            token_piece.lexeme.ends_with('\n') || token_piece.lexeme.ends_with('\r');

        let multi_line_token_index = self.buffer.len();
        self.buffer.push(token_piece);
        if !line_break_map.is_empty() {
            self.multi_line_map
                .insert(multi_line_token_index, line_break_map);
        }
        // A line is committed only once the token stream has actually crossed
        // a physical line boundary.
        if newline_count > 0 {
            // This clears `multi_line_start_line`
            self.commit_current_line();
            // We only need to insert one `Line::continued(start_line)` per
            // *interior* line:
            //
            // standalone           | local int i = /* Now we start long comment
            // continued            | with three line breaks and *exactly* two
            // continued            | inner lines that contain nothing but
            // continued_with_tokens| comment bytes! */ = 0;
            let inner_lines_count = newline_count - 1;
            for _ in 0..inner_lines_count {
                self.lines.push(Line::continued(start_line));
            }
            self.multi_line_start_line = if ends_with_newline {
                None
            } else {
                Some(start_line)
            };
        }

        self.line_number = start_line + newline_count;
    }

    /// Commits the tokens of the current physical line into `self.lines`.
    fn commit_current_line(&mut self) {
        let slice_end = self.buffer.len();
        // A trailing newline terminates the current physical line rather than
        // creating an additional empty line entry.
        if slice_end > self.uncommitted_start_index {
            let slice = self.uncommitted_start_index..slice_end;

            // If we were in the middle of a multi-line token, we
            // *always* consume `multi_line_start` here, ensuring that each call
            // to `commit_current_line()` only applies it once.
            // This guarantees no "bleed" between adjacent multi-line tokens.
            if let Some(from) = self.multi_line_start_line.take() {
                self.lines.push(Line::continued_with_tokens(from, slice));
            } else {
                self.lines.push(Line::standalone(slice));
            }
            self.uncommitted_start_index = slice_end;
        }
    }

    /// Runs post-processing that needs the token to already have a stable
    /// [`TokenPosition`].
    fn after_token(&mut self, position: TokenPosition) {
        let token_piece = self.buffer[position.0];

        if token_piece.token == Token::Error {
            self.had_errors = true;
            self.diagnostics
                .push(diagnostic_invalid_token(position, token_piece.lexeme));
        }

        self.delimiter_matcher.observe(
            position,
            token_piece.token,
            self.indent_tracker.current_indent(),
        );

        self.indent_tracker.observe_token(token_piece);
    }

    /// Finishes tokenization, converting accumulated data into
    /// [`TokenizedFile`].
    fn into_tokenized_file(mut self) -> TokenizedFile<'src> {
        // Commits the final line when the file does not end with a newline.
        self.commit_current_line();
        // If we still have a `multi_line_start`
        // (i.e. a pure multi-line token with no local tokens on its last line),
        // push a bare `Line::continued` entry.
        if let Some(from) = self.multi_line_start_line.take() {
            self.lines.push(Line::continued(from));
        }

        let (delimiter_matches, delimiter_diagnostics) = self.delimiter_matcher.finish();
        if !delimiter_diagnostics.is_empty() {
            self.had_errors = true;
        }
        self.diagnostics.extend(delimiter_diagnostics);

        self.buffer.shrink_to_fit();
        self.lines.shrink_to_fit();

        TokenizedFile {
            buffer: self.buffer,
            lines: self.lines,
            multi_line_map: self.multi_line_map,
            delimiter_matches,
            diagnostics: self.diagnostics,
            had_errors: self.had_errors,
        }
    }
}

fn make_token_data(token: Token, text: &str) -> TokenData<'_> {
    let length_utf16 = text.encode_utf16().count();
    TokenData {
        lexeme: text,
        token,
        utf16_length: length_utf16,
    }
}

/// Returns byte ranges of visible text characters for each physical line
/// spanned by `text`.
///
/// Returns an empty vector if `text` contains no line breaks.
fn split_visible_line_segments(text: &str) -> Vec<Range<usize>> {
    let bytes = text.as_bytes();
    let mut segments = Vec::new();

    let mut segment_start = 0usize;
    let mut saw_line_break = false;
    let mut bytes_iterator = bytes.iter().enumerate().peekable();
    while let Some((next_byte_index, &next_byte)) = bytes_iterator.next() {
        // Logos' regex rule is "\r\n|\n|\r", so we agree with it on new line
        // character treatment
        match next_byte {
            b'\r' => {
                saw_line_break = true;
                let visible_end = next_byte_index;
                let next_start =
                    if let Some((next_line_break_index, b'\n')) = bytes_iterator.peek().copied() {
                        bytes_iterator.next(); // consume '\n' of `\r\n`
                        next_line_break_index + 1
                    } else {
                        next_byte_index + 1
                    };
                segments.push(segment_start..visible_end);
                segment_start = next_start;
            }
            b'\n' => {
                saw_line_break = true;
                let visible_end = next_byte_index;
                segments.push(segment_start..visible_end);
                segment_start = next_byte_index + 1;
            }
            _ => (),
        }
    }
    // If the token contained at least one line break, include the visible
    // segment of its final physical line as well. This may be empty, e.g.
    // for text ending with '\n' or '\r\n'.
    if saw_line_break {
        segments.push(segment_start..bytes.len());
    }
    segments
}

// Because once `cursor` moves past the end of `buffer`, it can never become
// valid again.
impl std::iter::FusedIterator for Tokens<'_, '_> {}

impl<'file, 'src> Tokens<'file, 'src> {
    /// Advances the iterator cursor by one token.
    const fn advance(&mut self) {
        self.cursor.0 += 1;
    }

    /// Creates a new iterator.
    const fn new(source_file: &'file TokenizedFile<'src>) -> Self {
        Self {
            source_file,
            cursor: TokenPosition(0),
        }
    }
}

impl<'src> Iterator for Tokens<'_, 'src> {
    type Item = (TokenPosition, TokenData<'src>);

    fn next(&mut self) -> Option<Self::Item> {
        if let Some(&token_piece) = self.source_file.buffer.get(self.cursor.0) {
            let position = self.cursor;
            self.advance();
            Some((position, token_piece))
        } else {
            None
        }
    }
}

impl<'file, 'src> IntoIterator for &'file TokenizedFile<'src> {
    type Item = (TokenPosition, TokenData<'src>);
    type IntoIter = Tokens<'file, 'src>;

    fn into_iter(self) -> Self::IntoIter {
        self.iter()
    }
}

/// Sparse map between matched delimiter tokens.
///
/// Stores both directions:
///
/// - opening delimiter -> closing delimiter
/// - closing delimiter -> opening delimiter
#[derive(Clone, Debug, Default, PartialEq, Eq)]
pub struct DelimiterMatches {
    mates: HashMap<TokenPosition, TokenPosition>,
}

impl DelimiterMatches {
    /// Returns the matching delimiter token for `position`, if known.
    #[must_use]
    pub fn mate_of(&self, position: TokenPosition) -> Option<TokenPosition> {
        self.mates.get(&position).copied()
    }

    fn insert_pair(&mut self, left: TokenPosition, right: TokenPosition) {
        self.mates.insert(left, right);
        self.mates.insert(right, left);
    }
}

#[derive(Clone, Copy, Debug, Hash, PartialEq, Eq)]
enum DelimiterKind {
    Parenthesis,
    Bracket,
    Brace,
}

#[derive(Clone, Copy, Debug, Hash, PartialEq, Eq)]
enum DelimiterSide {
    Open,
    Close,
}

#[derive(Clone, Copy, Debug)]
struct OpenDelimiter {
    kind: DelimiterKind,
    position: TokenPosition,
    indent: usize,
}

#[derive(Debug, Default)]
struct DelimiterMatcher {
    stack: Vec<OpenDelimiter>,
    matches: DelimiterMatches,
    diagnostics: Vec<Diagnostic>,
    stopped_at_defaultproperties: bool,
}

impl DelimiterMatcher {
    fn handle_defaultproperties_boundary(&mut self, defaultproperties_position: TokenPosition) {
        while let Some(open) = self.stack.pop() {
            self.diagnostics
                .push(diagnostic_unclosed_delimiter_before_defaultproperties(
                    open,
                    defaultproperties_position,
                ));
        }
    }

    fn observe(&mut self, position: TokenPosition, token: Token, indent: usize) {
        if self.stopped_at_defaultproperties {
            return;
        }

        if token == Token::Keyword(Keyword::DefaultProperties) {
            self.handle_defaultproperties_boundary(position);
            self.stopped_at_defaultproperties = true;
            return;
        }

        let Some((side, kind)) = delimiter_of(token) else {
            return;
        };

        match side {
            DelimiterSide::Open => {
                self.stack.push(OpenDelimiter {
                    kind,
                    position,
                    indent,
                });
            }
            DelimiterSide::Close => {
                self.handle_close(kind, position, indent);
            }
        }
    }

    fn handle_close(
        &mut self,
        close_kind: DelimiterKind,
        close_position: TokenPosition,
        close_indent: usize,
    ) {
        let Some(top) = self.stack.last().copied() else {
            self.diagnostics
                .push(diagnostic_unexpected_closing_delimiter(
                    close_kind,
                    close_position,
                ));
            return;
        };

        if top.kind == close_kind {
            let open = self.stack.pop().expect("stack top was just checked");
            self.matches.insert_pair(open.position, close_position);
            return;
        }

        let earlier_same_kind = self.stack.iter().rposition(|open| open.kind == close_kind);

        match earlier_same_kind {
            Some(index) => {
                let bad_open = top;
                let recovered_open = self.stack[index];

                self.diagnostics
                    .push(diagnostic_unclosed_delimiter_before_later_close(
                        bad_open,
                        close_kind,
                        close_position,
                        recovered_open,
                        recovered_open.indent == close_indent,
                    ));

                while self.stack.len() - 1 > index {
                    self.stack.pop();
                }

                let open = self
                    .stack
                    .pop()
                    .expect("same-kind delimiter was found on the stack");

                self.matches.insert_pair(open.position, close_position);
            }
            None => {
                let bad_open = self.stack.pop().expect("stack top was known to exist");

                self.diagnostics
                    .push(diagnostic_mismatched_closing_delimiter(
                        bad_open,
                        close_kind,
                        close_position,
                    ));
            }
        }
    }

    fn finish(mut self) -> (DelimiterMatches, Vec<Diagnostic>) {
        if !self.stopped_at_defaultproperties {
            while let Some(open) = self.stack.pop() {
                self.diagnostics
                    .push(diagnostic_unclosed_delimiter_at_eof(open));
            }
        }

        (self.matches, self.diagnostics)
    }
}

fn delimiter_of(token: Token) -> Option<(DelimiterSide, DelimiterKind)> {
    match token {
        Token::LeftParenthesis => Some((DelimiterSide::Open, DelimiterKind::Parenthesis)),
        Token::RightParenthesis => Some((DelimiterSide::Close, DelimiterKind::Parenthesis)),

        Token::LeftBracket => Some((DelimiterSide::Open, DelimiterKind::Bracket)),
        Token::RightBracket => Some((DelimiterSide::Close, DelimiterKind::Bracket)),

        Token::LeftBrace => Some((DelimiterSide::Open, DelimiterKind::Brace)),
        Token::RightBrace => Some((DelimiterSide::Close, DelimiterKind::Brace)),

        _ => None,
    }
}

fn open_delimiter_text(kind: DelimiterKind) -> &'static str {
    match kind {
        DelimiterKind::Parenthesis => "(",
        DelimiterKind::Bracket => "[",
        DelimiterKind::Brace => "{",
    }
}

fn close_delimiter_text(kind: DelimiterKind) -> &'static str {
    match kind {
        DelimiterKind::Parenthesis => ")",
        DelimiterKind::Bracket => "]",
        DelimiterKind::Brace => "}",
    }
}

/// Crude indentation tracker used only for delimiter recovery diagnostics.
///
/// This is intentionally simple. It assumes indentation is represented by a
/// run of whitespace tokens before the first non-trivia token on a physical
/// line. It does not try to normalize tabs or handle every multiline trivia
/// edge case precisely, because indentation is only a diagnostic heuristic.
#[derive(Clone, Copy, Debug)]
struct IndentTracker {
    current_indent: usize,
    before_first_nontrivia_on_line: bool,
}

impl IndentTracker {
    fn new() -> Self {
        Self {
            current_indent: 0,
            before_first_nontrivia_on_line: true,
        }
    }

    fn current_indent(&self) -> usize {
        self.current_indent
    }

    fn observe_token(&mut self, token_piece: TokenData<'_>) {
        if token_is_newline(token_piece.token) {
            self.current_indent = 0;
            self.before_first_nontrivia_on_line = true;
            return;
        }

        if self.before_first_nontrivia_on_line && token_is_whitespace(token_piece.token) {
            self.current_indent += token_piece.lexeme.chars().count();
            return;
        }

        if token_is_trivia(token_piece.token) {
            return;
        }

        self.before_first_nontrivia_on_line = false;
    }
}

fn token_is_newline(token: Token) -> bool {
    matches!(token, Token::Newline)
}

fn token_is_whitespace(token: Token) -> bool {
    matches!(token, Token::Whitespace)
}

fn token_is_trivia(token: Token) -> bool {
    matches!(
        token,
        Token::Whitespace | Token::Newline | Token::LineComment | Token::BlockComment
    )
}

fn diagnostic_invalid_token(position: TokenPosition, lexeme: &str) -> Diagnostic {
    let shown = display_lexeme(lexeme);

    DiagnosticBuilder::error(format!("invalid token: {}", shown))
        .code("L0001")
        .primary_label(
            TokenSpan::new(position),
            format!("invalid token: {}", shown),
        )
        .build()
}

fn diagnostic_unexpected_closing_delimiter(
    close_kind: DelimiterKind,
    close_position: TokenPosition,
) -> Diagnostic {
    let close = close_delimiter_text(close_kind);

    DiagnosticBuilder::error(format!("unexpected closing delimiter: `{}`", close))
        .code("L0002")
        .primary_label(
            TokenSpan::new(close_position),
            "unexpected closing delimiter",
        )
        .build()
}

fn diagnostic_unclosed_delimiter_before_later_close(
    bad_open: OpenDelimiter,
    close_kind: DelimiterKind,
    close_position: TokenPosition,
    recovered_open: OpenDelimiter,
    same_indent: bool,
) -> Diagnostic {
    let bad_open_text = open_delimiter_text(bad_open.kind);
    let close_text = close_delimiter_text(close_kind);
    let recovered_open_text = open_delimiter_text(recovered_open.kind);

    let mut builder =
        DiagnosticBuilder::error(format!("unclosed delimiter before `{}`", close_text))
            .code("L0003")
            .primary_label(
                TokenSpan::new(bad_open.position),
                format!(
                    "this `{}` is not closed before `{}`",
                    bad_open_text, close_text
                ),
            )
            .secondary_label(
                TokenSpan::new(close_position),
                format!(
                    "this `{}` is matched with the earlier `{}`",
                    close_text, recovered_open_text
                ),
            );

    if same_indent {
        builder = builder.secondary_label(
            TokenSpan::new(recovered_open.position),
            format!(
                "this `{}` is likely the intended match",
                recovered_open_text,
            ),
        );
    }

    builder.build()
}

fn diagnostic_mismatched_closing_delimiter(
    bad_open: OpenDelimiter,
    close_kind: DelimiterKind,
    close_position: TokenPosition,
) -> Diagnostic {
    let open = open_delimiter_text(bad_open.kind);
    let close = close_delimiter_text(close_kind);

    DiagnosticBuilder::error(format!("mismatched closing delimiter: `{}`", close))
        .code("L0004")
        .primary_label(
            TokenSpan::new(close_position),
            format!("closing delimiter does not match `{}`", open),
        )
        .secondary_label(
            TokenSpan::new(bad_open.position),
            format!("`{}` opened here", open),
        )
        .build()
}

fn diagnostic_unclosed_delimiter_at_eof(open: OpenDelimiter) -> Diagnostic {
    let open_text = open_delimiter_text(open.kind);

    DiagnosticBuilder::error(format!("unclosed delimiter: `{}`", open_text))
        .code("L0005")
        .primary_label(
            TokenSpan::new(open.position),
            format!("this `{}` was never closed", open_text),
        )
        .build()
}

fn diagnostic_unclosed_delimiter_before_defaultproperties(
    open: OpenDelimiter,
    defaultproperties_position: TokenPosition,
) -> Diagnostic {
    let open_text = open_delimiter_text(open.kind);

    DiagnosticBuilder::error("unclosed delimiter before `defaultproperties`")
        .code("L0006")
        .primary_label(
            TokenSpan::new(open.position),
            format!("this `{}` is not closed before `defaultproperties`", open_text),
        )
        .secondary_label(
            TokenSpan::new(defaultproperties_position),
            "delimiter matching stops at `defaultproperties`",
        )
        .build()
}

fn display_lexeme(lexeme: &str) -> String {
    if lexeme.is_empty() {
        return "<empty>".to_string();
    }

    if lexeme == "`" {
        return "backtick".to_string();
    }

    let escaped: String = lexeme.chars().flat_map(char::escape_default).collect();

    const MAX_DISPLAY_CHARS: usize = 32;

    let mut display = String::new();
    let mut chars = escaped.chars();

    for _ in 0..MAX_DISPLAY_CHARS {
        let Some(character) = chars.next() else {
            return format!("`{}`", display);
        };
        display.push(character);
    }

    if chars.next().is_some() {
        display.push_str("...");
    }

    format!("`{}`", display)
}