[#3] TokenStream now hold generic variants.

This makes it so that the TokenStream and all it's associated Token
types use a generic when dealing with the variant of the Token.

Span was also given the ability to merge with another span. This
will make it easier to track the span as users group TokenTypes
together to make their domain specific types.

All tests and examples were updated for this change.

The version was incremented to 0.2.0.

tests/lexer_tests.rs

@@ -15,16 +15,22 @@ struct TestCase<'a>
 
 
 
-fn dummy_transform(tokens: &TokenStream) -> Vec<(TokenType, String)>
+fn dummy_transform(tokens: &TokenStream<TokenType>) -> TokenStream<TokenType>
 {
-   let mut new_tokens = Vec::new();
+   /*
+   let mut stream: TokenStream<(TokenType, String)> = TokenStream::default();
 
-   for token in tokens
+   stream.lexemes = tokens.lexemes.clone();
+   stream.locations = tokens.locations.clone();
+
+   for 0..tokens.lexemes.len()
    {
-      new_tokens.push((*token.variant, token.lexeme.to_string()));
+      stream.variants
    }
 
-   new_tokens
+   stream
+   */
+   tokens.clone()
 }
 
 fn write_temp_file(name: &str, content: &str) -> PathBuf
@@ -53,6 +59,8 @@ fn test_basic_lexing()
       Lexer::scan_text("magic runes", dummy_transform).expect("Lexer should \
                                                                succeed");
 
+   let tokens = tokens.into_iter().map(|t| { (*t.variant, String::from(t.lexeme))}).collect::<Vec<_>>();
+
    let expected = vec![(TokenType::Text, "magic".to_string()),
                        (TokenType::Whitespace, " ".to_string()),
                        (TokenType::Text, "runes".to_string()),
@@ -69,6 +77,8 @@ fn test_symbols_and_numbers()
       Lexer::scan_text("13 + 37", dummy_transform).expect("Lexer should \
                                                            succeed");
 
+   let tokens = tokens.into_iter().map(|t| { (*t.variant, String::from(t.lexeme))}).collect::<Vec<_>>();
+
    let expected = vec![(TokenType::Numeric, "13".into()),
                        (TokenType::Whitespace, " ".into()),
                        (TokenType::Symbol, "+".into()),
@@ -119,14 +129,14 @@ fn test_lexer_with_cases()
                                                               on case '{}'",
                                                              case.name));
 
-      let result_stripped: Vec<(TokenType, String)> = result;
+      let result = result.into_iter().map(|t| { (*t.variant, String::from(t.lexeme))}).collect::<Vec<_>>();
 
       let expected = case.expected
                          .iter()
                          .map(|(ty, s)| (*ty, s.to_string()))
                          .collect::<Vec<_>>();
 
-      assert_eq!(result_stripped, expected,
+      assert_eq!(result, expected,
                  "Mismatch in test case '{}'",
                  case.name);