Added the needed pop functionality to TokenStream.

src/token.rs

@@ -238,6 +238,20 @@ impl<T> TokenStream<T>
       self.locations.push(span);
    }
 
+   /// Removes and returns the last token from the stream, if any.
+   ///
+   /// # Returns
+   /// `Some((lexeme, variant, span))` if there was a token to pop,
+   /// or `None` if the stream was empty.
+   pub fn pop(&mut self) -> Option<(String, T, Span)>
+   {
+      let lexeme = self.lexemes.pop()?;
+      let variant = self.variants.pop()?;
+      let span = self.locations.pop()?;
+
+      Some((lexeme, variant, span))
+   }
+
    /// Returns a mutable iterator over the tokens in the stream.
    ///
    /// # Returns
@@ -262,13 +276,31 @@ impl<'a, T> IntoIterator for &'a TokenStream<T>
    }
 }
 
+impl<'a, T> TokenStreamIter<'a, T>
+{
+   pub fn peek(&self, n: usize) -> Option<Token<'a, T>>
+   {
+      let i = self.index + n;
+      if i < self.stream.len()
+      {
+         Some(Token { lexeme: &self.stream.lexemes[i],
+                      variant: &self.stream.variants[i],
+                      span: &self.stream.locations[i] })
+      }
+      else
+      {
+         None
+      }
+   }
+}
+
 impl<'a, T> Iterator for TokenStreamIter<'a, T>
 {
    type Item = Token<'a, T>;
 
    fn next(&mut self) -> Option<Self::Item>
    {
-      if self.index < self.stream.lexemes.len()
+      if self.index < self.stream.len()
       {
          let i = self.index;
          self.index += 1;