ecsolticia.codeberg.page
Minimal Imperative Parsing Library | https://docs.rs/mipl
set up foundations for subparsers
+1
-2
examples/calc.rs
+1
-2
examples/calc.rs
+15
src/lexer.rs
+15
src/lexer.rs
···
12
12
DelimitersParam,
13
13
};
14
14
}
15
+
/// The lexer prelude.
16
+
pub mod prelude {
17
+
pub use super::*;
18
+
pub use super::del::*;
19
+
}
15
20
16
21
/// Type for marking expected [Token::End] cases.
17
22
#[derive(Debug, PartialEq, Clone)]
···
49
54
));
50
55
}
51
56
57
+
/// Add a token to the list ad-hoc.
58
+
pub fn add_token(&mut self, token: Token) {
59
+
self.0.push_back(token)
60
+
}
61
+
62
+
/// Get a new empty list of tokens.
63
+
pub fn new_empty() -> Tokens {
64
+
Tokens(VecDeque::new())
65
+
}
66
+
52
67
/// Tokenize string input.
53
68
pub fn new<T: AsRef<str>>(
54
69
value: T,
+16
-1
src/lib.rs
+16
-1
src/lib.rs
+1
-5
src/parser.rs
+1
-5
src/parser.rs
+26
-9
src/parser/parsers.rs
+26
-9
src/parser/parsers.rs
···
5
5
6
6
/// The peeking functionality of concrete parsers.
7
7
///
8
-
pub trait PeekerOperator<Item> {
8
+
pub trait PeekerOperator {
9
+
/// The type of the elements.
10
+
type Item;
11
+
9
12
/// Returns the `Item` if it matches some pattern as
10
13
/// defined by the implementation of this function
11
14
/// for any particular parser.
···
13
16
/// Returns `None` for non-matching tokens. This behavior
14
17
/// is used to automatically derive [Operator] implementation
15
18
/// over certain generics.
16
-
fn peek_for(&self, parser: &mut Parser) -> Option<Item>;
19
+
fn peek_for(&self, parser: &mut Parser) -> Option<Self::Item>;
17
20
}
18
21
22
+
/// Marks container input types at the trait-level.
23
+
pub trait IsContainerInput {}
24
+
/// Marks container types at the trait-level.
25
+
pub trait ContainedType {}
26
+
27
+
/// Interface for containing data in the concrete parser structs.
19
28
pub trait Container {
20
-
type Input;
21
-
type ContainedType;
29
+
/// The type the concrete parsers are constructed from.
30
+
type Input: IsContainerInput;
31
+
/// The type the concrete parser structs contain.
32
+
type ContainedType: ContainedType;
22
33
34
+
/// Created a new instance of the type to be contained.
23
35
fn new_contained(value: Self::Input) -> Self::ContainedType;
36
+
/// Created a new parser struct based on the expected input type.
24
37
fn new(value: Self::Input) -> Self;
25
38
}
26
39
···
28
41
/// of concrete parsers.
29
42
///
30
43
/// Has blanket implementation for all concrete parsers.
31
-
pub trait Operator<Item>: PeekerOperator<Item> + Container {
44
+
pub trait Operator: PeekerOperator + Container {
32
45
/// Peek the next token in the [Parser] stream. If it
33
46
/// matches the pattern specified by the [PeekerOperator]
34
47
/// implementation, return something of type `Item` relevant
35
48
/// to the token.
36
49
///
37
50
/// Return type is the same as that of [PeekerOperator::peek_for].
38
-
fn try_next(&self, parser: &mut Parser) -> Option<Item>;
51
+
fn try_next(&self, parser: &mut Parser) -> Option<Self::Item>;
39
52
}
40
53
41
-
impl<T, Item> Operator<Item> for T
54
+
impl<T> Operator for T
42
55
where
43
-
T: PeekerOperator<Item> + Container
56
+
T: PeekerOperator + Container
44
57
{
45
-
fn try_next(&self, parser: &mut Parser) -> Option<Item> {
58
+
fn try_next(&self, parser: &mut Parser) -> Option<Self::Item> {
46
59
if let Some(itm) = self.peek_for(parser) {
47
60
parser.next();
48
61
Some(itm)
···
52
65
}
53
66
}
54
67
68
+
impl IsContainerInput for String {}
69
+
impl ContainedType for String {}
70
+
impl IsContainerInput for () {}
71
+
impl ContainedType for () {}
55
72
mod exact_match;
56
73
mod or_exact_match;
57
74
mod any_str_match;
+3
-1
src/parser/parsers/any_str_match.rs
+3
-1
src/parser/parsers/any_str_match.rs
···
5
5
/// In particular, match the [Token::Str] variant of
6
6
/// the [Token] enum, and return the inner [String].
7
7
pub struct AnyStrMatch;
8
-
impl PeekerOperator<String> for AnyStrMatch {
8
+
impl PeekerOperator for AnyStrMatch {
9
+
type Item = String;
10
+
9
11
fn peek_for(&self, parser: &mut Parser) -> Option<String> {
10
12
match parser.peek()? {
11
13
Token::Str(s) => Some(s),
+3
-1
src/parser/parsers/exact_match.rs
+3
-1
src/parser/parsers/exact_match.rs
···
5
5
/// The string to match to.
6
6
pub matches: String
7
7
}
8
-
impl PeekerOperator<String> for ExactMatch {
8
+
impl PeekerOperator for ExactMatch {
9
+
type Item = String;
10
+
9
11
fn peek_for(&self, parser: &mut Parser) -> Option<String> {
10
12
if let Some(tok) = parser.peek() {
11
13
match tok {
+3
-1
src/parser/parsers/is_newline.rs
+3
-1
src/parser/parsers/is_newline.rs
···
4
4
5
5
/// Match a newline.
6
6
pub struct IsNewline;
7
-
impl PeekerOperator<NewlineToken> for IsNewline {
7
+
impl PeekerOperator for IsNewline {
8
+
type Item = NewlineToken;
9
+
8
10
fn peek_for(&self, parser: &mut Parser) -> Option<NewlineToken> {
9
11
if let Token::Newline(NewlineToken) = parser.peek()? {
10
12
Some(NewlineToken)
+7
-1
src/parser/parsers/or_exact_match.rs
+7
-1
src/parser/parsers/or_exact_match.rs
···
6
6
pub struct OrExactMatch {
7
7
matches: HashSet<String>
8
8
}
9
-
impl PeekerOperator<String> for OrExactMatch {
9
+
impl PeekerOperator for OrExactMatch {
10
+
type Item = String;
11
+
10
12
fn peek_for(&self, parser: &mut Parser) -> Option<String> {
11
13
if let Token::Str(s) = parser.peek()? {
12
14
if self.matches.contains(&s) {
···
19
21
}
20
22
}
21
23
}
24
+
25
+
impl IsContainerInput for Vec<&'static str> {}
26
+
impl ContainedType for HashSet<String> {}
27
+
22
28
impl Container for OrExactMatch {
23
29
type ContainedType = HashSet<String>;
24
30
type Input = Vec<&'static str>;
+21
-2
src/parser/subparser.rs
+21
-2
src/parser/subparser.rs
···
1
-
//! [Parser] consume parts of other [Parser]'s tokens.
1
+
//! [Parsers](Parser) that consume parts of other [Parser]'s tokens.
2
+
3
+
use crate::prelude::*;
4
+
5
+
/// Functionality to build blanket [Parsers](Parser) using
6
+
/// a parent [Parser] by consuming its
7
+
/// tokens based on some pattern.
8
+
///
9
+
/// The pattern depends on the generic `C`, which corresponds
10
+
/// to a concrete parser. (See [parsers].)
11
+
pub trait ConcreteSubparser<C> {
12
+
/// Build the subparser based on the concrete parser.
13
+
fn subparse
14
+
(value: C::Input, parser: &mut Parser) -> Parser
15
+
where
16
+
C: Container;
17
+
}
2
18
3
-
use super::*;
19
+
mod collect_till;
20
+
pub use {
21
+
collect_till::CollectTill,
22
+
};
+36
src/parser/subparser/collect_till.rs
+36
src/parser/subparser/collect_till.rs
···
1
+
use super::*;
2
+
3
+
/// Consume tokens into a new parser until the concrete
4
+
/// parser returns `None`.
5
+
///
6
+
/// Said concrete pattern here
7
+
/// is the generic parameter `Op`.
8
+
pub struct CollectTill<Op: parsers::Operator>(Op);
9
+
impl<Op> ConcreteSubparser<Op> for CollectTill<Op>
10
+
where
11
+
Op: parsers::Operator
12
+
{
13
+
fn subparse
14
+
(value: Op::Input, parser: &mut Parser) -> Parser
15
+
where
16
+
Op: Container
17
+
{
18
+
let op = Op::new(value);
19
+
20
+
let mut tokens: Tokens = Tokens::new_empty();
21
+
22
+
loop {
23
+
if let Some(_tok) = op.try_next(parser) {
24
+
break;
25
+
} else {
26
+
if let Some(tok) = parser.next() {
27
+
tokens.add_token(tok);
28
+
} else {
29
+
break;
30
+
}
31
+
}
32
+
}
33
+
34
+
Parser::new(tokens)
35
+
}
36
+
}
+19
tests/common.rs
+19
tests/common.rs
···
1
+
use mipl::{
2
+
lexer::prelude::*,
3
+
parser::*
4
+
};
5
+
6
+
pub fn setup_space_seps_parser(source: &str) -> Parser {
7
+
let d_vec: Vec<char> = vec![' '];
8
+
let d_del = DiscardDelimiters::new(d_vec);
9
+
let k_vec: Vec<char> = vec![];
10
+
let k_del = KeepDelimiters::new(k_vec);
11
+
let del_param = DelimitersParam {
12
+
keep: k_del, discard: d_del
13
+
};
14
+
15
+
Parser::from(
16
+
source,
17
+
del_param
18
+
)
19
+
}
+1
-3
tests/test_first_emphasized_in_line.rs
+1
-3
tests/test_first_emphasized_in_line.rs
+3
-17
tests/test_parsers.rs
+3
-17
tests/test_parsers.rs
···
1
-
use mipl::parser::{*, parsers::*};
2
-
use mipl::lexer::del::*;
3
-
use mipl::lexer::*;
4
-
5
-
fn setup_space_seps_parser(source: &str) -> Parser {
6
-
let d_vec: Vec<char> = vec![' '];
7
-
let d_del = DiscardDelimiters::new(d_vec);
8
-
let k_vec: Vec<char> = vec![];
9
-
let k_del = KeepDelimiters::new(k_vec);
10
-
let del_param = DelimitersParam {
11
-
keep: k_del, discard: d_del
12
-
};
1
+
use mipl::prelude::*;
13
2
14
-
Parser::from(
15
-
source,
16
-
del_param
17
-
)
18
-
}
3
+
mod common;
4
+
use common::*;
19
5
20
6
#[test]
21
7
fn test_exact_match() {
+48
tests/test_subparser.rs
+48
tests/test_subparser.rs
···
1
+
use mipl::prelude::*;
2
+
3
+
mod common;
4
+
use common::*;
5
+
6
+
#[test]
7
+
fn test_collect_till_exact_match_subparser() {
8
+
let mut parser = setup_space_seps_parser(
9
+
"a b c d e f g h"
10
+
);
11
+
let known_subparser = setup_space_seps_parser(
12
+
"a b c d"
13
+
);
14
+
15
+
let mut subparser = CollectTill::<ExactMatch>::subparse(
16
+
"e".to_string(),
17
+
&mut parser
18
+
);
19
+
20
+
for k in known_subparser {
21
+
if let Token::Newline(NewlineToken) = k {
22
+
break;
23
+
}
24
+
assert_eq!(Some(k), subparser.next())
25
+
}
26
+
}
27
+
28
+
#[test]
29
+
fn test_collect_till_is_newline() {
30
+
let mut parser = setup_space_seps_parser(
31
+
"a b\n c d e f g h"
32
+
);
33
+
let known_subparser = setup_space_seps_parser(
34
+
"a b"
35
+
);
36
+
37
+
let mut subparser = CollectTill::<IsNewline>::subparse(
38
+
(),
39
+
&mut parser
40
+
);
41
+
42
+
for k in known_subparser {
43
+
if let Token::Newline(NewlineToken) = k {
44
+
break;
45
+
}
46
+
assert_eq!(Some(k), subparser.next())
47
+
}
48
+
}