ecsolticia.codeberg.page
Minimal Imperative Parsing Library | https://docs.rs/mipl
add simple whitespace/newline-separating lexer
+68
src/lexer.rs
+68
src/lexer.rs
···
1
+
//! Lexer for `mipl`.
2
+
3
+
/// Type for marking expected [Token::End] cases.
4
+
#[derive(Debug, PartialEq)]
5
+
pub struct EndToken;
6
+
/// Type for marking expected [Token::Newline] cases.
7
+
#[derive(Debug, PartialEq)]
8
+
pub struct NewlineToken;
9
+
10
+
/// The core token type.
11
+
#[derive(Debug, PartialEq)]
12
+
pub enum Token {
13
+
Newline(NewlineToken),
14
+
End(EndToken),
15
+
Str(String)
16
+
}
17
+
impl Token {
18
+
fn from<T: AsRef<str>>(value: T) -> Token {
19
+
Token::Str(String::from(
20
+
value.as_ref()
21
+
))
22
+
}
23
+
}
24
+
25
+
use std::collections::VecDeque;
26
+
/// Iterator of tokens, and the tokenizer.
27
+
pub struct Tokens(VecDeque<Token>);
28
+
impl Tokens {
29
+
/// Tokenize string input.
30
+
fn new<T: AsRef<str>>(value: T) -> Tokens {
31
+
let mut inner = VecDeque::<Token>::new();
32
+
33
+
for line in value.as_ref().lines() {
34
+
35
+
for tok in line.split_whitespace() {
36
+
inner.push_back(Token::from(tok));
37
+
}
38
+
39
+
inner.push_back(Token::Newline(NewlineToken));
40
+
}
41
+
inner.push_back(Token::End(EndToken));
42
+
43
+
Tokens(inner)
44
+
}
45
+
}
46
+
47
+
#[cfg(test)]
48
+
mod tests {
49
+
use super::*;
50
+
51
+
#[test]
52
+
fn test_tokens_new() {
53
+
let mut toks = Tokens::new("A line.\nA quick brown fox.");
54
+
55
+
toks.0.pop_front();
56
+
toks.0.pop_front();
57
+
let newline = toks.0.pop_front().unwrap();
58
+
toks.0.pop_front();
59
+
60
+
let val = match toks.0.pop_front().unwrap() {
61
+
Token::Str(s) => s,
62
+
_ => panic!("Expected string.")
63
+
};
64
+
65
+
assert_eq!("quick", val);
66
+
assert_eq!(Token::Newline(NewlineToken), newline);
67
+
}
68
+
}
+3
-13
src/lib.rs
+3
-13
src/lib.rs
···
1
-
pub fn add(left: u64, right: u64) -> u64 {
2
-
left + right
3
-
}
4
-
5
-
#[cfg(test)]
6
-
mod tests {
7
-
use super::*;
1
+
//! `mipl`: Minimal imperative parsing library
8
2
9
-
#[test]
10
-
fn it_works() {
11
-
let result = add(2, 2);
12
-
assert_eq!(result, 4);
13
-
}
14
-
}
3
+
pub mod lexer;
4
+
pub mod parser;
+1
src/parser.rs
+1
src/parser.rs
···
1
+
//! The Parser.