rune/examples/basic.rs

use std::path::PathBuf;

use rune::{Lexer, Span, TokenStream, TokenType};



#[derive(Debug, Clone, PartialEq, Eq)]
pub enum MarkdownTokenType {
    Heading(u8),
    EmphasisStart,
    EmphasisEnd,
    StrongStart,
    StrongEnd,
    CodeSpan,
    Text,
    Symbol,
    Whitespace,
    Newline,
    Unknown,
}


impl std::fmt::Display for MarkdownTokenType {
    fn fmt(&self, f: &mut std::fmt::Formatter<'_>) -> std::fmt::Result {
        match self {
            MarkdownTokenType::Heading(level) => write!(f, "Heading({})", level),
            MarkdownTokenType::EmphasisStart => write!(f, "EmphasisStart"),
            MarkdownTokenType::EmphasisEnd => write!(f, "EmphasisEnd"),
            MarkdownTokenType::StrongStart => write!(f, "StrongStart"),
            MarkdownTokenType::StrongEnd => write!(f, "StrongEnd"),
            MarkdownTokenType::CodeSpan => write!(f, "CodeSpan"),
            MarkdownTokenType::Text => write!(f, "Text"),
            MarkdownTokenType::Symbol => write!(f, "Symbol"),
            MarkdownTokenType::Whitespace => write!(f, "Whitespace"),
            MarkdownTokenType::Newline => write!(f, "Newline"),
            MarkdownTokenType::Unknown => write!(f, "Unknown"),
        }
    }
}



// Define how you want to interpret base tokens
pub fn transform(input: &TokenStream<TokenType>) -> TokenStream<MarkdownTokenType> {
    let mut output = TokenStream::new();

    let mut i = 0;
    while i < input.len() {
        let token = input.get(i).unwrap(); // safe due to bounds check above

        match token.variant {
            TokenType::Symbol if token.lexeme == "#" => {
                // Count consecutive #s for heading level
                let mut level = 1;
                let mut span = token.span.clone();

                while i + 1 < input.len() {
                    let next = input.get(i + 1).unwrap();
                    if *next.variant == TokenType::Symbol && next.lexeme == "#" {
                        level += 1;
                        span.end = next.span.end;
                        i += 1;
                    } else {
                        break;
                    }
                }

                output.push(token.lexeme.repeat(level),
                            MarkdownTokenType::Heading(level as u8),
                            span);
            }

            TokenType::Symbol if token.lexeme == "*" => {
                // Look ahead to see if it's strong (**) or emphasis (*)
                if i + 1 < input.len() {
                    let next = input.get(i + 1).unwrap();
                    if *next.variant == TokenType::Symbol && next.lexeme == "*" {
                        output.push("**".to_string(),
                                    MarkdownTokenType::StrongStart,
                                    Span::merge(*token.span, *next.span));
                        i += 1; // skip the second '*'
                    } else {
                        output.push("*".to_string(),
                                    MarkdownTokenType::EmphasisStart,
                                    token.span.clone());
                    }
                } else {
                    output.push("*".to_string(),
                                MarkdownTokenType::EmphasisStart,
                                token.span.clone());
                }
            }

            TokenType::Symbol if token.lexeme == "`" => {
                output.push(token.lexeme.to_string(),
                            MarkdownTokenType::CodeSpan,
                            token.span.clone());
            }

            TokenType::Text => {
                output.push(token.lexeme.to_string(),
                            MarkdownTokenType::Text,
                            token.span.clone());
            }

            TokenType::Symbol => {
                output.push(token.lexeme.to_string(),
                            MarkdownTokenType::Symbol,
                            token.span.clone());
            }

            TokenType::Whitespace => {
                output.push(token.lexeme.to_string(),
                            MarkdownTokenType::Whitespace,
                            token.span.clone());
            }

            TokenType::Newline => {
                output.push(token.lexeme.to_string(),
                            MarkdownTokenType::Newline,
                            token.span.clone());
            }

            _ => {
                output.push(token.lexeme.to_string(),
                            MarkdownTokenType::Unknown,
                            token.span.clone());
            }
        }

        i += 1;
    }

    output
}


fn main() -> Result<(), Box<dyn std::error::Error>>
{
   let mut path = PathBuf::from(env!("CARGO_MANIFEST_DIR"));
   path.push("examples/example.md");

   let tokens = Lexer::scan_file(path, transform)?;

   // The tuple here is from the transform functions return type.
   for token in &tokens
   {
      println!("{}", token);
   }

   Ok(())
}