+7

src/lexer.rs

··· 59 59 self.0.push_back(token) 60 60 } 61 61 62 62 + /// Get [Tokens] from a vector of [Token] items. 63 63 + pub fn from_vec(toks: Vec<Token>) -> Tokens { 64 64 + Tokens( 65 65 + VecDeque::from(toks) 66 66 + ) 67 67 + } 68 68 + 62 69 /// Get a new empty list of tokens. 63 70 pub fn new_empty() -> Tokens { 64 71 Tokens(VecDeque::new())

+25 -1

src/parser.rs

··· 14 14 self.tokens.peek().cloned() 15 15 } 16 16 17 17 + /// Extend the parser with [Tokens]. 18 18 + /// 19 19 + /// Current implementation *could be* 20 20 + /// costly as it relies on cloning. 21 21 + pub fn extend(&mut self, tokens: Tokens) { 22 22 + let mut toks_buf = Tokens::new_empty(); 23 23 + 24 24 + for token in self.clone() { 25 25 + toks_buf.add_token(token); 26 26 + } 27 27 + 28 28 + for token in tokens.clone() { 29 29 + toks_buf.add_token(token); 30 30 + } 31 31 + 32 32 + self.tokens = tokens.peekable(); 33 33 + } 34 34 + 35 35 + /// Get a new empty [Parser]. 36 36 + pub fn new_empty() -> Parser { 37 37 + let tokens = Tokens::new_empty(); 38 38 + Self::new(tokens) 39 39 + } 40 40 + 17 41 /// Create a new [Parser] from tokens. 18 42 pub fn new(tokens: Tokens) -> Parser { 19 43 Parser { tokens: tokens.peekable() } ··· 39 63 } 40 64 41 65 pub mod concrete_parser; 42 42 - pub mod subparser; 66 66 + pub mod subparser;

+13 -1

src/parser/subparser.rs

··· 16 16 C: concrete_parser::ContainerCp; 17 17 } 18 18 19 19 + /// Functionality to build blanket [Parsers](Parser) using 20 20 + /// a parent [Parser] by consuming a list of tokens from it. 21 21 + pub trait VariadicConcreteSubparser<C> { 22 22 + /// Build the subparser based on the concrete parser. 23 23 + fn subparse 24 24 + (values: Vec<C::Input>, parser: &mut Parser) -> Parser 25 25 + where 26 26 + C: concrete_parser::ContainerCp; 27 27 + } 28 28 + 19 29 mod collect_till; 20 30 mod collect_while; 31 31 + mod collect_if_sequence; 21 32 pub use { 22 33 collect_till::CollectTill, 23 34 collect_while::CollectWhile, 24 24 - }; 35 35 + collect_if_sequence::{CollectIfSeq, CollectIfExactSeq}, 36 36 + };

+67

src/parser/subparser/collect_if_sequence.rs

··· 1 1 + use crate::prelude::*; 2 2 + 3 3 + /// Consume a sequence of tokens into a new parser while the 4 4 + /// concrete parsers return something. Stop consuming once 5 5 + /// all values are exhausted, or if one of then returned 6 6 + /// `None`. 7 7 + pub struct CollectIfSeq<Cp: ConcreteParser>(Cp); 8 8 + impl<Cp> VariadicConcreteSubparser<Cp> for CollectIfSeq<Cp> 9 9 + where 10 10 + Cp: ConcreteParser 11 11 + { 12 12 + fn subparse 13 13 + (values: Vec<Cp::Input>, parser: &mut Parser) -> Parser 14 14 + where 15 15 + Cp: concrete_parser::ContainerCp 16 16 + { 17 17 + let mut tokens = Tokens::new_empty(); 18 18 + 19 19 + for v in values { 20 20 + let cp = Cp::new(v); 21 21 + 22 22 + if let Some(token) = cp.try_next_token(parser) { 23 23 + tokens.add_token(token); 24 24 + } 25 25 + } 26 26 + 27 27 + Parser::new(tokens) 28 28 + } 29 29 + } 30 30 + 31 31 + /// Consume a sequence of tokens into a new parser **if** 32 32 + /// the coming sequence of tokens match a sequence. 33 33 + pub struct CollectIfExactSeq<Cp: ConcreteParser>(Cp); 34 34 + impl<Cp> VariadicConcreteSubparser<Cp> for CollectIfExactSeq<Cp> 35 35 + where 36 36 + Cp: ConcreteParser 37 37 + { 38 38 + fn subparse 39 39 + (values: Vec<Cp::Input>, parser: &mut Parser) -> Parser 40 40 + where 41 41 + Cp: concrete_parser::ContainerCp 42 42 + { 43 43 + let mut toks_buffer: Tokens = Tokens::new_empty(); 44 44 + 45 45 + let mut exact: bool = true; 46 46 + 47 47 + for v in values { 48 48 + let cp = Cp::new(v); 49 49 + 50 50 + if let Some(token) = cp.try_next_token(parser) { 51 51 + toks_buffer.add_token(token) 52 52 + } else { 53 53 + if let Some(token) = parser.next() { 54 54 + toks_buffer.add_token(token); 55 55 + } 56 56 + exact = false; 57 57 + } 58 58 + } 59 59 + 60 60 + if exact { 61 61 + Parser::new(toks_buffer) 62 62 + } else { 63 63 + parser.extend(toks_buffer); 64 64 + Parser::new_empty() 65 65 + } 66 66 + } 67 67 + }

+1 -4

src/parser/subparser/collect_while.rs

··· 3 3 /// Consume tokens into a new parser while the concrete 4 4 /// parser returns something. Stop consuming if it returns 5 5 /// `None`. 6 6 - /// 7 7 - /// Said concrete pattern here is provided by 8 8 - /// the generic parameter `Op`. 9 6 pub struct CollectWhile<Op: ConcreteParser>(Op); 10 7 impl<Op> ConcreteSubparser<Op> for CollectWhile<Op> 11 8 where ··· 30 27 31 28 Parser::new(tokens) 32 29 } 33 33 - } 30 30 + }

+131

tests/test_subparser.rs

··· 3 3 mod common; 4 4 use common::*; 5 5 6 6 + /// Assert that two parsers successively yield a certain number 7 7 + /// `n` of the same token. 8 8 + fn assert_parsers_same_till( 9 9 + mut a: Parser, 10 10 + mut b: Parser, 11 11 + n: usize 12 12 + ) { 13 13 + let mut i: usize = 0; 14 14 + while i < n { 15 15 + assert_eq!(a.next(), b.next()); 16 16 + i += 1; 17 17 + } 18 18 + } 19 19 + 20 20 + fn not_assert_parsers_same_till( 21 21 + mut a: Parser, 22 22 + mut b: Parser, 23 23 + n: usize 24 24 + ) { 25 25 + let mut i: usize = 0; 26 26 + let mut not: bool = false; 27 27 + while i < n { 28 28 + if a.next() != b.next() { 29 29 + not = true; 30 30 + break; 31 31 + } 32 32 + i += i; 33 33 + } 34 34 + assert!(not) 35 35 + } 36 36 + 6 37 #[test] 7 38 fn test_collect_till_exact_match_subparser() { 8 39 let mut parser = setup_space_seps_parser( ··· 64 95 assert_eq!(known_subparser.next(), subparser.next()); 65 96 i += 1; 66 97 } 98 98 + } 99 99 + 100 100 + #[test] 101 101 + fn test_collect_if_sequence() { 102 102 + let mut parser = setup_space_seps_parser( 103 103 + "a b c d e f g h i j" 104 104 + ); 105 105 + let known_subparser = setup_space_seps_parser( 106 106 + "a b c d" 107 107 + ); 108 108 + 109 109 + let values: Vec<String> = vec!["a", "b", "c", "d", "e"] 110 110 + .into_iter() 111 111 + .map(String::from) 112 112 + .collect(); 113 113 + 114 114 + let subparser = CollectIfSeq::<ExactMatch>::subparse( 115 115 + values, 116 116 + &mut parser 117 117 + ); 118 118 + 119 119 + assert_parsers_same_till( 120 120 + known_subparser, 121 121 + subparser, 122 122 + 4 123 123 + ); 124 124 + } 125 125 + 126 126 + #[test] 127 127 + fn test_collect_if_exact_sequence() { 128 128 + let mut parser = setup_space_seps_parser( 129 129 + "a b c d e f g h i j" 130 130 + ); 131 131 + let known_subparser = setup_space_seps_parser( 132 132 + "a b" 133 133 + ); 134 134 + 135 135 + let values: Vec<String> = vec!["a", "b", "c", "d"] 136 136 + .into_iter() 137 137 + .map(String::from) 138 138 + .collect(); 139 139 + 140 140 + let subparser = CollectIfExactSeq::<ExactMatch>::subparse( 141 141 + values, 142 142 + &mut parser 143 143 + ); 144 144 + 145 145 + assert_parsers_same_till( 146 146 + known_subparser, 147 147 + subparser, 148 148 + 2 149 149 + ); 150 150 + } 151 151 + 152 152 + #[test] 153 153 + fn test_fail_collect_if_exact_sequence() { 154 154 + let mut parser = setup_space_seps_parser( 155 155 + "a b c d e f g h i j" 156 156 + ); 157 157 + let known_subparser = setup_space_seps_parser( 158 158 + "a b c d" 159 159 + ); 160 160 + 161 161 + let values: Vec<String> = vec!["a", "b", "c", "d", "f"] 162 162 + .into_iter() 163 163 + .map(String::from) 164 164 + .collect(); 165 165 + 166 166 + let subparser = CollectIfExactSeq::<ExactMatch>::subparse( 167 167 + values, 168 168 + &mut parser 169 169 + ); 170 170 + 171 171 + not_assert_parsers_same_till( 172 172 + known_subparser, 173 173 + subparser, 174 174 + 4 175 175 + ); 176 176 + } 177 177 + 178 178 + #[test] 179 179 + fn test_fail_collect_if_exact_keeps_parent_parser_intact() { 180 180 + let mut parser = setup_space_seps_parser( 181 181 + "a b c d" 182 182 + ); 183 183 + let initial_parser = parser.clone(); 184 184 + 185 185 + let values: Vec<String> = vec!["a", "b", "c", "e"] 186 186 + .into_iter() 187 187 + .map(String::from) 188 188 + .collect(); 189 189 + 190 190 + let _subparser = CollectIfExactSeq::<ExactMatch>::subparse( 191 191 + values, 192 192 + &mut parser 193 193 + ); 194 194 + 195 195 + println!("{:#?}", parser); 196 196 + 197 197 + assert_parsers_same_till(parser, initial_parser, 4); 67 198 }