ecsolticia.codeberg.page
Minimal Imperative Parsing Library | https://docs.rs/mipl
make it possible to keep or discard newline delimiters
+44
-27
+17
-1
examples/calc.rs
+17
-1
examples/calc.rs
···
53
53
}
54
54
55
55
fn setup_parser(source: String) -> Parser {
56
-
let d_vec: Vec<char> = vec![' '];
56
+
let d_vec: Vec<char> = vec![' ', '\n', '\t'];
57
57
let d_del = DiscardDelimiters::new(d_vec);
58
58
let k_vec: Vec<char> = vec!['+', '-', '*', '/', '(', ')'];
59
59
let k_del = KeepDelimiters::new(k_vec);
···
132
132
let mut parser = setup_parser("2 * (3 + 2) * 2".to_string());
133
133
let res = eval(&mut parser);
134
134
assert_eq!(Some(20.0), res);
135
+
}
136
+
137
+
#[test]
138
+
fn test_newline() {
139
+
let mut parser = setup_parser("2 * \n (3 + 2)".to_string());
140
+
println!("{:#?}", parser);
141
+
let res = eval(&mut parser);
142
+
assert_eq!(Some(10.0), res);
143
+
}
144
+
145
+
#[test]
146
+
fn test_tab() {
147
+
let mut parser = setup_parser("2 * \t (3 + 2)".to_string());
148
+
println!("{:#?}", parser);
149
+
let res= eval(&mut parser);
150
+
assert_eq!(Some(10.0), res)
135
151
}
136
152
}
+26
-25
src/lexer.rs
+26
-25
src/lexer.rs
···
54
54
));
55
55
}
56
56
57
+
fn char_into_tok(ch: char) -> Token {
58
+
match ch {
59
+
'\n' => Token::Newline(NewlineToken),
60
+
_ => Token::Str(String::from(ch))
61
+
}
62
+
}
63
+
57
64
/// Add a token to the list ad-hoc.
58
65
pub fn add_token(&mut self, token: Token) {
59
66
self.0.push_back(token)
···
78
85
) -> Tokens {
79
86
let mut inner = VecDeque::<Token>::new();
80
87
81
-
for line in value.as_ref().lines() {
88
+
let mut chars_buf: Vec<char> = Vec::new();
82
89
83
-
let mut chars_buf: Vec<char> = Vec::new();
90
+
let chars = value.as_ref().chars();
91
+
92
+
for ch in chars {
93
+
let is_d_del = delimiters.discard.contains(ch);
94
+
let is_k_del = delimiters.keep.contains(ch);
95
+
let is_del = is_d_del | is_k_del;
84
96
85
-
let chars = line.chars();
97
+
if is_del {
98
+
if !chars_buf.is_empty() {
99
+
Tokens::chars_buf_into_tok(&mut chars_buf, &mut inner);
100
+
}
86
101
87
-
for ch in chars {
88
-
let is_d_del = delimiters.discard.contains(ch);
89
-
let is_k_del = delimiters.keep.contains(ch);
90
-
let is_del = is_d_del | is_k_del;
91
-
92
-
if is_del {
93
-
if !chars_buf.is_empty() {
94
-
Tokens::chars_buf_into_tok(&mut chars_buf, &mut inner);
95
-
}
96
-
97
-
if is_k_del {
98
-
inner.push_back(Token::Str(
99
-
String::from(ch)
100
-
));
101
-
}
102
-
} else {
103
-
chars_buf.push(ch)
102
+
if is_k_del {
103
+
inner.push_back(Tokens::char_into_tok(ch))
104
104
}
105
+
} else {
106
+
chars_buf.push(ch)
105
107
}
108
+
}
106
109
107
-
if !chars_buf.is_empty() {
110
+
if !chars_buf.is_empty() {
108
111
Tokens::chars_buf_into_tok(&mut chars_buf, &mut inner);
109
-
}
112
+
}
110
113
111
-
inner.push_back(Token::Newline(NewlineToken));
112
-
}
113
114
inner.push_back(Token::End(EndToken));
114
115
115
116
Tokens(inner)
···
131
132
fn test_tokens_new() {
132
133
let d_dels = DiscardDelimiters::new( vec![' '] );
133
134
let del_params = DelimitersParam{
134
-
keep: KeepDelimiters::new( vec![] ),
135
+
keep: KeepDelimiters::new( vec!['\n'] ),
135
136
discard: d_dels
136
137
};
137
138
+1
-1
tests/common.rs
+1
-1
tests/common.rs
···
6
6
pub fn setup_space_seps_parser(source: &str) -> Parser {
7
7
let d_vec: Vec<char> = vec![' '];
8
8
let d_del = DiscardDelimiters::new(d_vec);
9
-
let k_vec: Vec<char> = vec![];
9
+
let k_vec: Vec<char> = vec!['\n'];
10
10
let k_del = KeepDelimiters::new(k_vec);
11
11
let del_param = DelimitersParam {
12
12
keep: k_del, discard: d_del