Myrddin Dundragon
693ff20224
Scanning a file has all the lines terminating with a '\n' newline character, but when giving the text directly to the lexer via the scan_text function it should not append a newline at the end if there was no newline in the original input.
249 lines
8.2 KiB
Rust
249 lines
8.2 KiB
Rust
use std::fs::File;
|
|
use std::io::{BufRead, BufReader};
|
|
|
|
use super::error::LexerError;
|
|
use super::position::{Position, Span};
|
|
use super::token::{TokenStream, TokenType};
|
|
|
|
|
|
|
|
/// The size of data chunks to read from a file. This is an arbitrary choice,
|
|
/// set to 1MB.
|
|
const BUFFER_SIZE: usize = 1024 * 1024;
|
|
|
|
|
|
|
|
/// The `Lexer` struct is responsible for performing lexical analysis
|
|
/// (tokenization) on text.
|
|
///
|
|
/// It processes input from a file or string character-by-character and
|
|
/// generates a stream of tokens, such as text, numbers, whitespace, symbols,
|
|
/// and newlines. These tokens are accumulated into a `TokenStream`, which is a
|
|
/// flat, cache-friendly data structure designed for efficient iteration.
|
|
///
|
|
/// After the base tokens are generated, the `Lexer` allows for transformation
|
|
/// of these tokens into richer, domain-specific types via a user-provided
|
|
/// `transform` function. This transformation can be used to convert base tokens
|
|
/// into specific elements of a Markdown syntax tree, custom DSL tokens, or any
|
|
/// other custom format you need.
|
|
///
|
|
/// # Example
|
|
///
|
|
/// ```rust
|
|
/// use rune::{Lexer, TokenStream, TokenType};
|
|
///
|
|
/// fn transform(tokens: &TokenStream<TokenType>) -> TokenStream<TokenType>
|
|
/// {
|
|
/// tokens.clone()
|
|
/// }
|
|
///
|
|
/// let tokens = Lexer::scan_text("Runes += 42", transform).unwrap();
|
|
///
|
|
/// for token in &tokens
|
|
/// {
|
|
/// println!("{}", token);
|
|
/// }
|
|
/// ```
|
|
///
|
|
/// # Design Considerations
|
|
/// - Utilizes a flat `TokenStream` to improve performance and reduce heap
|
|
/// overhead.
|
|
/// - Consolidates contiguous characters into compound tokens (e.g., multi-digit
|
|
/// numbers).
|
|
/// - Extensible via the `transform` function, enabling the creation of
|
|
/// higher-level constructs, like Markdown elements or syntax trees for a
|
|
/// custom DSL.
|
|
///
|
|
/// # Error Handling
|
|
/// The lexer will return a `LexerError` if the input file cannot be opened or
|
|
/// read. Errors include issues such as missing files, read failures, or invalid
|
|
/// input formats.
|
|
pub enum Lexer {}
|
|
|
|
|
|
|
|
impl Lexer
|
|
{
|
|
/// Scans a file and generates a vector of transformed tokens based on the
|
|
/// provided `transform` function.
|
|
///
|
|
/// This method opens a file from the given `path`, reads the file line by
|
|
/// line, and converts the input into a stream of tokens. The tokens are
|
|
/// then passed to the `transform` function, which allows users to map
|
|
/// base tokens into domain-specific types.
|
|
///
|
|
/// # Parameters
|
|
/// - `path`: A path to the file to be lexically analyzed.
|
|
/// - `transform`: A function that takes a `TokenStream<TokenType>` and
|
|
/// transforms it into a `TokenStream<T>` where `T` is a domain-specific
|
|
/// type.
|
|
///
|
|
/// # Returns
|
|
/// A `Result<TokenStream<T>, LexerError>` where `T` is the transformed token
|
|
/// type, or an error.
|
|
///
|
|
/// # Errors
|
|
/// Returns a `LexerError` if the file cannot be opened or read.
|
|
pub fn scan_file<P, F, T>(path: P, transform: F)
|
|
-> Result<TokenStream<T>, LexerError>
|
|
where P: AsRef<std::path::Path>,
|
|
F: FnOnce(&TokenStream<TokenType>) -> TokenStream<T>
|
|
{
|
|
let mut cursor = Position::default();
|
|
let mut stream = TokenStream::new();
|
|
|
|
let input_file = File::open(&path).map_err(|err| {
|
|
LexerError::new(
|
|
"Unable to open file for Lexical Analysis.",
|
|
Span::default(),
|
|
Some(path.as_ref().to_path_buf()),
|
|
None).with_source(err)
|
|
})?;
|
|
|
|
let reader = BufReader::with_capacity(BUFFER_SIZE, input_file);
|
|
|
|
// Read the file line by line.
|
|
for line in reader.lines()
|
|
{
|
|
match line
|
|
{
|
|
Ok(text) =>
|
|
{
|
|
Self::scan(&text, &mut stream, &mut cursor);
|
|
}
|
|
Err(_) =>
|
|
{
|
|
return Err(LexerError::new("Unable to read line during \
|
|
Lexical Analysis.",
|
|
Span::default(),
|
|
Some(path.as_ref()
|
|
.to_string_lossy()
|
|
.to_string()),
|
|
None));
|
|
}
|
|
}
|
|
|
|
// Add the newline token after each line.
|
|
stream.push("\n".to_string(),
|
|
TokenType::Newline,
|
|
Span::with_single(cursor));
|
|
cursor.line += 1;
|
|
cursor.column = 0;
|
|
}
|
|
|
|
Ok(transform(&stream))
|
|
}
|
|
|
|
/// Scans a full in-memory string and produces transformed tokens.
|
|
///
|
|
/// This method tokenizes the input string `text` and returns the transformed
|
|
/// tokens using the provided `transform` function. It's a convenient way
|
|
/// to perform lexical analysis on in-memory strings without needing to
|
|
/// read from a file.
|
|
///
|
|
/// # Parameters
|
|
/// - `text`: A string slice representing the in-memory input text to
|
|
/// analyze.
|
|
/// - `transform`: A function that transforms the base tokens into
|
|
/// domain-specific types.
|
|
///
|
|
/// # Returns
|
|
/// A `Result<TokenStream<T>, LexerError>` where `T` is the transformed token
|
|
/// type, or an error.
|
|
pub fn scan_text<F, T>(text: &str, transform: F)
|
|
-> Result<TokenStream<T>, LexerError>
|
|
where F: FnOnce(&TokenStream<TokenType>) -> TokenStream<T>
|
|
{
|
|
let mut cursor = Position::default();
|
|
let mut stream = TokenStream::new();
|
|
|
|
// Process each line in the input string.
|
|
for line in text.lines()
|
|
{
|
|
Self::scan(line, &mut stream, &mut cursor);
|
|
|
|
// Add the newline token after each line.
|
|
stream.push("\n".to_string(),
|
|
TokenType::Newline,
|
|
Span::with_single(cursor));
|
|
cursor.line += 1;
|
|
cursor.column = 0;
|
|
}
|
|
|
|
// Remove the last newline character if the text did not end with a
|
|
// newline.
|
|
if !text.ends_with('\n')
|
|
{
|
|
stream.pop();
|
|
}
|
|
|
|
Ok(transform(&stream))
|
|
}
|
|
|
|
/// Internal method that scans a single line of text into tokens.
|
|
///
|
|
/// This method processes each character of a line and generates the
|
|
/// corresponding token. It handles cases like numeric tokens, text
|
|
/// tokens, symbols, and whitespace.
|
|
///
|
|
/// # Parameters
|
|
/// - `line`: A line of text to be lexically analyzed.
|
|
/// - `stream`: A mutable reference to the token stream where the generated
|
|
/// tokens will be pushed.
|
|
/// - `cursor`: A mutable reference to the cursor position, which tracks the
|
|
/// current position in the input.
|
|
fn scan(line: &str, stream: &mut TokenStream<TokenType>,
|
|
cursor: &mut Position)
|
|
{
|
|
for c in line.chars()
|
|
{
|
|
// Get the token type based on the character.
|
|
let variant = get_token_type(c);
|
|
let last = stream.len().saturating_sub(1);
|
|
|
|
// Handle token merging for contiguous tokens like numbers or text.
|
|
if !stream.is_empty() &&
|
|
variant == stream.variants[last] &&
|
|
(variant == TokenType::Numeric || variant == TokenType::Text)
|
|
{
|
|
stream.lexemes[last].push(c);
|
|
stream.locations[last].end = *cursor;
|
|
}
|
|
else
|
|
{
|
|
// Add a new token to the stream.
|
|
stream.push(c.to_string(), variant, Span::with_single(*cursor));
|
|
}
|
|
|
|
cursor.column += 1;
|
|
}
|
|
}
|
|
}
|
|
|
|
|
|
|
|
/// Determines the type of a token based on the current character.
|
|
///
|
|
/// This helper function is responsible for identifying whether the current
|
|
/// character is part of a known token type such as numeric, text, whitespace,
|
|
/// or symbol.
|
|
///
|
|
/// # Parameters
|
|
/// - `curr_char`: The current character to analyze.
|
|
///
|
|
/// # Returns
|
|
/// A `TokenType` corresponding to the character type (e.g., `Numeric`, `Text`,
|
|
/// `Whitespace`, etc.).
|
|
fn get_token_type(curr_char: char) -> TokenType
|
|
{
|
|
match curr_char
|
|
{
|
|
'\n' => TokenType::Newline,
|
|
c if c.is_whitespace() => TokenType::Whitespace,
|
|
c if c.is_numeric() => TokenType::Numeric,
|
|
c if c.is_alphabetic() => TokenType::Text,
|
|
c if !c.is_whitespace() && !c.is_alphanumeric() => TokenType::Symbol,
|
|
_ => TokenType::Unknown
|
|
}
|
|
}
|