Just some basic updating and cleaning up.
- Added comments. - Ran cargo fmt. - Updated the versioning.
This commit is contained in:
@ -5,133 +5,159 @@ use rune::{Lexer, Span, TokenStream, TokenType};
|
||||
|
||||
|
||||
#[derive(Debug, Clone, PartialEq, Eq)]
|
||||
pub enum MarkdownTokenType {
|
||||
Heading(u8),
|
||||
EmphasisStart,
|
||||
EmphasisEnd,
|
||||
StrongStart,
|
||||
StrongEnd,
|
||||
CodeSpan,
|
||||
Text,
|
||||
Symbol,
|
||||
Whitespace,
|
||||
Newline,
|
||||
Unknown,
|
||||
pub enum MarkdownTokenType
|
||||
{
|
||||
Heading(u8),
|
||||
EmphasisStart,
|
||||
EmphasisEnd,
|
||||
StrongStart,
|
||||
StrongEnd,
|
||||
CodeSpan,
|
||||
Text,
|
||||
Symbol,
|
||||
Whitespace,
|
||||
Newline,
|
||||
Unknown
|
||||
}
|
||||
|
||||
|
||||
impl std::fmt::Display for MarkdownTokenType {
|
||||
fn fmt(&self, f: &mut std::fmt::Formatter<'_>) -> std::fmt::Result {
|
||||
match self {
|
||||
MarkdownTokenType::Heading(level) => write!(f, "Heading({})", level),
|
||||
MarkdownTokenType::EmphasisStart => write!(f, "EmphasisStart"),
|
||||
MarkdownTokenType::EmphasisEnd => write!(f, "EmphasisEnd"),
|
||||
MarkdownTokenType::StrongStart => write!(f, "StrongStart"),
|
||||
MarkdownTokenType::StrongEnd => write!(f, "StrongEnd"),
|
||||
MarkdownTokenType::CodeSpan => write!(f, "CodeSpan"),
|
||||
MarkdownTokenType::Text => write!(f, "Text"),
|
||||
MarkdownTokenType::Symbol => write!(f, "Symbol"),
|
||||
MarkdownTokenType::Whitespace => write!(f, "Whitespace"),
|
||||
MarkdownTokenType::Newline => write!(f, "Newline"),
|
||||
MarkdownTokenType::Unknown => write!(f, "Unknown"),
|
||||
}
|
||||
}
|
||||
impl std::fmt::Display for MarkdownTokenType
|
||||
{
|
||||
fn fmt(&self, f: &mut std::fmt::Formatter<'_>) -> std::fmt::Result
|
||||
{
|
||||
match self
|
||||
{
|
||||
MarkdownTokenType::Heading(level) => write!(f, "Heading({})", level),
|
||||
MarkdownTokenType::EmphasisStart => write!(f, "EmphasisStart"),
|
||||
MarkdownTokenType::EmphasisEnd => write!(f, "EmphasisEnd"),
|
||||
MarkdownTokenType::StrongStart => write!(f, "StrongStart"),
|
||||
MarkdownTokenType::StrongEnd => write!(f, "StrongEnd"),
|
||||
MarkdownTokenType::CodeSpan => write!(f, "CodeSpan"),
|
||||
MarkdownTokenType::Text => write!(f, "Text"),
|
||||
MarkdownTokenType::Symbol => write!(f, "Symbol"),
|
||||
MarkdownTokenType::Whitespace => write!(f, "Whitespace"),
|
||||
MarkdownTokenType::Newline => write!(f, "Newline"),
|
||||
MarkdownTokenType::Unknown => write!(f, "Unknown")
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
|
||||
|
||||
// Define how you want to interpret base tokens
|
||||
pub fn transform(input: &TokenStream<TokenType>) -> TokenStream<MarkdownTokenType> {
|
||||
let mut output = TokenStream::new();
|
||||
pub fn transform(input: &TokenStream<TokenType>)
|
||||
-> TokenStream<MarkdownTokenType>
|
||||
{
|
||||
let mut output = TokenStream::new();
|
||||
|
||||
let mut i = 0;
|
||||
while i < input.len() {
|
||||
let token = input.get(i).unwrap(); // safe due to bounds check above
|
||||
let mut i = 0;
|
||||
while i < input.len()
|
||||
{
|
||||
let token = input.get(i).unwrap(); // safe due to bounds check above
|
||||
|
||||
match token.variant {
|
||||
TokenType::Symbol if token.lexeme == "#" => {
|
||||
// Count consecutive #s for heading level
|
||||
let mut level = 1;
|
||||
let mut span = token.span.clone();
|
||||
match token.variant
|
||||
{
|
||||
TokenType::Symbol if token.lexeme == "#" =>
|
||||
{
|
||||
// Count consecutive #s for heading level
|
||||
let mut level = 1;
|
||||
let mut span = token.span.clone();
|
||||
|
||||
while i + 1 < input.len() {
|
||||
let next = input.get(i + 1).unwrap();
|
||||
if *next.variant == TokenType::Symbol && next.lexeme == "#" {
|
||||
level += 1;
|
||||
span.end = next.span.end;
|
||||
i += 1;
|
||||
} else {
|
||||
break;
|
||||
}
|
||||
}
|
||||
|
||||
output.push(token.lexeme.repeat(level),
|
||||
MarkdownTokenType::Heading(level as u8),
|
||||
span);
|
||||
while i + 1 < input.len()
|
||||
{
|
||||
let next = input.get(i + 1).unwrap();
|
||||
if *next.variant == TokenType::Symbol && next.lexeme == "#"
|
||||
{
|
||||
level += 1;
|
||||
span.end = next.span.end;
|
||||
i += 1;
|
||||
}
|
||||
else
|
||||
{
|
||||
break;
|
||||
}
|
||||
}
|
||||
|
||||
TokenType::Symbol if token.lexeme == "*" => {
|
||||
// Look ahead to see if it's strong (**) or emphasis (*)
|
||||
if i + 1 < input.len() {
|
||||
let next = input.get(i + 1).unwrap();
|
||||
if *next.variant == TokenType::Symbol && next.lexeme == "*" {
|
||||
output.push("**".to_string(),
|
||||
MarkdownTokenType::StrongStart,
|
||||
Span::merge(*token.span, *next.span));
|
||||
i += 1; // skip the second '*'
|
||||
} else {
|
||||
output.push("*".to_string(),
|
||||
MarkdownTokenType::EmphasisStart,
|
||||
token.span.clone());
|
||||
}
|
||||
} else {
|
||||
output.push("*".to_string(),
|
||||
MarkdownTokenType::EmphasisStart,
|
||||
token.span.clone());
|
||||
}
|
||||
output.push(token.lexeme.repeat(level),
|
||||
MarkdownTokenType::Heading(level as u8),
|
||||
span);
|
||||
}
|
||||
|
||||
TokenType::Symbol if token.lexeme == "*" =>
|
||||
{
|
||||
// Look ahead to see if it's strong (**) or emphasis (*)
|
||||
if i + 1 < input.len()
|
||||
{
|
||||
let next = input.get(i + 1).unwrap();
|
||||
if *next.variant == TokenType::Symbol && next.lexeme == "*"
|
||||
{
|
||||
output.push("**".to_string(),
|
||||
MarkdownTokenType::StrongStart,
|
||||
Span::merge(*token.span, *next.span));
|
||||
i += 1; // skip the second '*'
|
||||
}
|
||||
else
|
||||
{
|
||||
output.push("*".to_string(),
|
||||
MarkdownTokenType::EmphasisStart,
|
||||
token.span.clone());
|
||||
}
|
||||
}
|
||||
|
||||
TokenType::Symbol if token.lexeme == "`" => {
|
||||
output.push(token.lexeme.to_string(),
|
||||
MarkdownTokenType::CodeSpan,
|
||||
token.span.clone());
|
||||
else
|
||||
{
|
||||
output.push("*".to_string(),
|
||||
MarkdownTokenType::EmphasisStart,
|
||||
token.span.clone());
|
||||
}
|
||||
}
|
||||
|
||||
TokenType::Text => {
|
||||
output.push(token.lexeme.to_string(),
|
||||
MarkdownTokenType::Text,
|
||||
token.span.clone());
|
||||
}
|
||||
TokenType::Symbol if token.lexeme == "`" =>
|
||||
{
|
||||
output.push(token.lexeme.to_string(),
|
||||
MarkdownTokenType::CodeSpan,
|
||||
token.span.clone());
|
||||
}
|
||||
|
||||
TokenType::Symbol => {
|
||||
output.push(token.lexeme.to_string(),
|
||||
MarkdownTokenType::Symbol,
|
||||
token.span.clone());
|
||||
}
|
||||
TokenType::Text =>
|
||||
{
|
||||
output.push(token.lexeme.to_string(),
|
||||
MarkdownTokenType::Text,
|
||||
token.span.clone());
|
||||
}
|
||||
|
||||
TokenType::Whitespace => {
|
||||
output.push(token.lexeme.to_string(),
|
||||
MarkdownTokenType::Whitespace,
|
||||
token.span.clone());
|
||||
}
|
||||
TokenType::Symbol =>
|
||||
{
|
||||
output.push(token.lexeme.to_string(),
|
||||
MarkdownTokenType::Symbol,
|
||||
token.span.clone());
|
||||
}
|
||||
|
||||
TokenType::Newline => {
|
||||
output.push(token.lexeme.to_string(),
|
||||
MarkdownTokenType::Newline,
|
||||
token.span.clone());
|
||||
}
|
||||
TokenType::Whitespace =>
|
||||
{
|
||||
output.push(token.lexeme.to_string(),
|
||||
MarkdownTokenType::Whitespace,
|
||||
token.span.clone());
|
||||
}
|
||||
|
||||
_ => {
|
||||
output.push(token.lexeme.to_string(),
|
||||
MarkdownTokenType::Unknown,
|
||||
token.span.clone());
|
||||
}
|
||||
}
|
||||
TokenType::Newline =>
|
||||
{
|
||||
output.push(token.lexeme.to_string(),
|
||||
MarkdownTokenType::Newline,
|
||||
token.span.clone());
|
||||
}
|
||||
|
||||
i += 1;
|
||||
}
|
||||
_ =>
|
||||
{
|
||||
output.push(token.lexeme.to_string(),
|
||||
MarkdownTokenType::Unknown,
|
||||
token.span.clone());
|
||||
}
|
||||
}
|
||||
|
||||
output
|
||||
i += 1;
|
||||
}
|
||||
|
||||
output
|
||||
}
|
||||
|
||||
|
||||
|
||||
Reference in New Issue
Block a user