ecsolticia.codeberg.page
Minimal Imperative Parsing Library | https://docs.rs/mipl
fix empty string tokens in lexer output
+17
-9
src/lexer.rs
+17
-9
src/lexer.rs
···
36
36
#[derive(Debug, Clone)]
37
37
pub struct Tokens(VecDeque<Token>);
38
38
impl Tokens {
39
+
fn chars_buf_into_tok(
40
+
chars_buf: &mut Vec<char>,
41
+
inner: &mut VecDeque<Token>
42
+
) {
43
+
let new_tok: String = chars_buf.iter().collect();
44
+
45
+
chars_buf.drain(..);
46
+
47
+
inner.push_back(Token::Str(
48
+
new_tok
49
+
));
50
+
}
51
+
39
52
/// Tokenize string input.
40
53
pub fn new<T: AsRef<str>>(
41
54
value: T,
···
55
68
let is_del = is_d_del | is_k_del;
56
69
57
70
if is_del {
58
-
let new_tok: String = chars_buf.iter().collect();
59
-
chars_buf = Vec::new();
60
-
inner.push_back(Token::Str(
61
-
new_tok
62
-
));
71
+
if !chars_buf.is_empty() {
72
+
Tokens::chars_buf_into_tok(&mut chars_buf, &mut inner);
73
+
}
63
74
64
75
if is_k_del {
65
76
inner.push_back(Token::Str(
···
72
83
}
73
84
74
85
if !chars_buf.is_empty() {
75
-
let new_tok: String = chars_buf.iter().collect();
76
-
inner.push_back(Token::Str(
77
-
new_tok
78
-
));
86
+
Tokens::chars_buf_into_tok(&mut chars_buf, &mut inner);
79
87
}
80
88
81
89
inner.push_back(Token::Newline(NewlineToken));