ecsolticia.codeberg.page
Minimal Imperative Parsing Library | https://docs.rs/mipl
fix empty string tokens in lexer output
+17
-9
+17
-9
src/lexer.rs
+17
-9
src/lexer.rs
···
36
#[derive(Debug, Clone)]
37
pub struct Tokens(VecDeque<Token>);
38
impl Tokens {
39
/// Tokenize string input.
40
pub fn new<T: AsRef<str>>(
41
value: T,
···
55
let is_del = is_d_del | is_k_del;
56
57
if is_del {
58
-
let new_tok: String = chars_buf.iter().collect();
59
-
chars_buf = Vec::new();
60
-
inner.push_back(Token::Str(
61
-
new_tok
62
-
));
63
64
if is_k_del {
65
inner.push_back(Token::Str(
···
72
}
73
74
if !chars_buf.is_empty() {
75
-
let new_tok: String = chars_buf.iter().collect();
76
-
inner.push_back(Token::Str(
77
-
new_tok
78
-
));
79
}
80
81
inner.push_back(Token::Newline(NewlineToken));
···
36
#[derive(Debug, Clone)]
37
pub struct Tokens(VecDeque<Token>);
38
impl Tokens {
39
+
fn chars_buf_into_tok(
40
+
chars_buf: &mut Vec<char>,
41
+
inner: &mut VecDeque<Token>
42
+
) {
43
+
let new_tok: String = chars_buf.iter().collect();
44
+
45
+
chars_buf.drain(..);
46
+
47
+
inner.push_back(Token::Str(
48
+
new_tok
49
+
));
50
+
}
51
+
52
/// Tokenize string input.
53
pub fn new<T: AsRef<str>>(
54
value: T,
···
68
let is_del = is_d_del | is_k_del;
69
70
if is_del {
71
+
if !chars_buf.is_empty() {
72
+
Tokens::chars_buf_into_tok(&mut chars_buf, &mut inner);
73
+
}
74
75
if is_k_del {
76
inner.push_back(Token::Str(
···
83
}
84
85
if !chars_buf.is_empty() {
86
+
Tokens::chars_buf_into_tok(&mut chars_buf, &mut inner);
87
}
88
89
inner.push_back(Token::Newline(NewlineToken));