ocrm.bsky.social
the game where you go into mines and start crafting! but for consoles (forked directly from smartcmd's github)
1// Boost token_functions.hpp ------------------------------------------------//
2
3// Copyright John R. Bandela 2001.
4
5// Distributed under the Boost Software License, Version 1.0. (See
6// accompanying file LICENSE_1_0.txt or copy at
7// http://www.boost.org/LICENSE_1_0.txt)
8
9// See http://www.boost.org/libs/tokenizer/ for documentation.
10
11// Revision History:
12// 01 Oct 2004 Joaquin M Lopez Munoz
13// Workaround for a problem with string::assign in msvc-stlport
14// 06 Apr 2004 John Bandela
15// Fixed a bug involving using char_delimiter with a true input iterator
16// 28 Nov 2003 Robert Zeh and John Bandela
17// Converted into "fast" functions that avoid using += when
18// the supplied iterator isn't an input_iterator; based on
19// some work done at Archelon and a version that was checked into
20// the boost CVS for a short period of time.
21// 20 Feb 2002 John Maddock
22// Removed using namespace std declarations and added
23// workaround for BOOST_NO_STDC_NAMESPACE (the library
24// can be safely mixed with regex).
25// 06 Feb 2002 Jeremy Siek
26// Added char_separator.
27// 02 Feb 2002 Jeremy Siek
28// Removed tabs and a little cleanup.
29
30
31#ifndef BOOST_TOKEN_FUNCTIONS_JRB120303_HPP_
32#define BOOST_TOKEN_FUNCTIONS_JRB120303_HPP_
33
34#include <vector>
35#include <stdexcept>
36#include <string>
37#include <cctype>
38#include <algorithm> // for find_if
39#include <boost/config.hpp>
40#include <boost/assert.hpp>
41#include <boost/detail/workaround.hpp>
42#include <boost/mpl/if.hpp>
43#if !defined(BOOST_NO_CWCTYPE)
44#include <cwctype>
45#endif
46
47//
48// the following must not be macros if we are to prefix them
49// with std:: (they shouldn't be macros anyway...)
50//
51#ifdef ispunct
52# undef ispunct
53#endif
54#ifdef iswpunct
55# undef iswpunct
56#endif
57#ifdef isspace
58# undef isspace
59#endif
60#ifdef iswspace
61# undef iswspace
62#endif
63//
64// fix namespace problems:
65//
66#ifdef BOOST_NO_STDC_NAMESPACE
67namespace std{
68 using ::ispunct;
69 using ::isspace;
70#if !defined(BOOST_NO_CWCTYPE)
71 using ::iswpunct;
72 using ::iswspace;
73#endif
74}
75#endif
76
77namespace boost{
78 //===========================================================================
79 // The escaped_list_separator class. Which is a model of TokenizerFunction
80 // An escaped list is a super-set of what is commonly known as a comma
81 // separated value (csv) list.It is separated into fields by a comma or
82 // other character. If the delimiting character is inside quotes, then it is
83 // counted as a regular character.To allow for embedded quotes in a field,
84 // there can be escape sequences using the \ much like C.
85 // The role of the comma, the quotation mark, and the escape
86 // character (backslash \), can be assigned to other characters.
87
88 struct escaped_list_error : public std::runtime_error{
89 escaped_list_error(const std::string& what_arg):std::runtime_error(what_arg) { }
90 };
91
92
93// The out of the box GCC 2.95 on cygwin does not have a char_traits class.
94// MSVC does not like the following typename
95 template <class Char,
96 class Traits = BOOST_DEDUCED_TYPENAME std::basic_string<Char>::traits_type >
97 class escaped_list_separator {
98
99 private:
100 typedef std::basic_string<Char,Traits> string_type;
101 struct char_eq {
102 Char e_;
103 char_eq(Char e):e_(e) { }
104 bool operator()(Char c) {
105 return Traits::eq(e_,c);
106 }
107 };
108 string_type escape_;
109 string_type c_;
110 string_type quote_;
111 bool last_;
112
113 bool is_escape(Char e) {
114 char_eq f(e);
115 return std::find_if(escape_.begin(),escape_.end(),f)!=escape_.end();
116 }
117 bool is_c(Char e) {
118 char_eq f(e);
119 return std::find_if(c_.begin(),c_.end(),f)!=c_.end();
120 }
121 bool is_quote(Char e) {
122 char_eq f(e);
123 return std::find_if(quote_.begin(),quote_.end(),f)!=quote_.end();
124 }
125 template <typename iterator, typename Token>
126 void do_escape(iterator& next,iterator end,Token& tok) {
127 if (++next == end)
128 throw escaped_list_error(std::string("cannot end with escape"));
129 if (Traits::eq(*next,'n')) {
130 tok+='\n';
131 return;
132 }
133 else if (is_quote(*next)) {
134 tok+=*next;
135 return;
136 }
137 else if (is_c(*next)) {
138 tok+=*next;
139 return;
140 }
141 else if (is_escape(*next)) {
142 tok+=*next;
143 return;
144 }
145 else
146 throw escaped_list_error(std::string("unknown escape sequence"));
147 }
148
149 public:
150
151 explicit escaped_list_separator(Char e = '\\',
152 Char c = ',',Char q = '\"')
153 : escape_(1,e), c_(1,c), quote_(1,q), last_(false) { }
154
155 escaped_list_separator(string_type e, string_type c, string_type q)
156 : escape_(e), c_(c), quote_(q), last_(false) { }
157
158 void reset() {last_=false;}
159
160 template <typename InputIterator, typename Token>
161 bool operator()(InputIterator& next,InputIterator end,Token& tok) {
162 bool bInQuote = false;
163 tok = Token();
164
165 if (next == end) {
166 if (last_) {
167 last_ = false;
168 return true;
169 }
170 else
171 return false;
172 }
173 last_ = false;
174 for (;next != end;++next) {
175 if (is_escape(*next)) {
176 do_escape(next,end,tok);
177 }
178 else if (is_c(*next)) {
179 if (!bInQuote) {
180 // If we are not in quote, then we are done
181 ++next;
182 // The last character was a c, that means there is
183 // 1 more blank field
184 last_ = true;
185 return true;
186 }
187 else tok+=*next;
188 }
189 else if (is_quote(*next)) {
190 bInQuote=!bInQuote;
191 }
192 else {
193 tok += *next;
194 }
195 }
196 return true;
197 }
198 };
199
200 //===========================================================================
201 // The classes here are used by offset_separator and char_separator to implement
202 // faster assigning of tokens using assign instead of +=
203
204 namespace tokenizer_detail {
205 //===========================================================================
206 // Tokenizer was broken for wide character separators, at least on Windows, since
207 // CRT functions isspace etc only expect values in [0, 0xFF]. Debug build asserts
208 // if higher values are passed in. The traits extension class should take care of this.
209 // Assuming that the conditional will always get optimized out in the function
210 // implementations, argument types are not a problem since both forms of character classifiers
211 // expect an int.
212
213#if !defined(BOOST_NO_CWCTYPE)
214 template<typename traits, int N>
215 struct traits_extension_details : public traits {
216 typedef typename traits::char_type char_type;
217 static bool isspace(char_type c)
218 {
219 return std::iswspace(c) != 0;
220 }
221 static bool ispunct(char_type c)
222 {
223 return std::iswpunct(c) != 0;
224 }
225 };
226
227 template<typename traits>
228 struct traits_extension_details<traits, 1> : public traits {
229 typedef typename traits::char_type char_type;
230 static bool isspace(char_type c)
231 {
232 return std::isspace(c) != 0;
233 }
234 static bool ispunct(char_type c)
235 {
236 return std::ispunct(c) != 0;
237 }
238 };
239#endif
240
241
242 // In case there is no cwctype header, we implement the checks manually.
243 // We make use of the fact that the tested categories should fit in ASCII.
244 template<typename traits>
245 struct traits_extension : public traits {
246 typedef typename traits::char_type char_type;
247 static bool isspace(char_type c)
248 {
249#if !defined(BOOST_NO_CWCTYPE)
250 return traits_extension_details<traits, sizeof(char_type)>::isspace(c);
251#else
252 return static_cast< unsigned >(c) <= 255 && std::isspace(c) != 0;
253#endif
254 }
255
256 static bool ispunct(char_type c)
257 {
258#if !defined(BOOST_NO_CWCTYPE)
259 return traits_extension_details<traits, sizeof(char_type)>::ispunct(c);
260#else
261 return static_cast< unsigned >(c) <= 255 && std::ispunct(c) != 0;
262#endif
263 }
264 };
265
266 // The assign_or_plus_equal struct contains functions that implement
267 // assign, +=, and clearing based on the iterator type. The
268 // generic case does nothing for plus_equal and clearing, while
269 // passing through the call for assign.
270 //
271 // When an input iterator is being used, the situation is reversed.
272 // The assign method does nothing, plus_equal invokes operator +=,
273 // and the clearing method sets the supplied token to the default
274 // token constructor's result.
275 //
276
277 template<class IteratorTag>
278 struct assign_or_plus_equal {
279 template<class Iterator, class Token>
280 static void assign(Iterator b, Iterator e, Token &t) {
281
282#if BOOST_WORKAROUND(BOOST_MSVC, < 1300) &&\
283 BOOST_WORKAROUND(__SGI_STL_PORT, < 0x500) &&\
284 defined(_STLP_DEBUG) &&\
285 (defined(_STLP_USE_DYNAMIC_LIB) || defined(_DLL))
286 // Problem with string::assign for msvc-stlport in debug mode: the
287 // linker tries to import the templatized version of this memfun,
288 // which is obviously not exported.
289 // See http://www.stlport.com/dcforum/DCForumID6/1763.html for details.
290
291 t = Token();
292 while(b != e) t += *b++;
293#else
294 t.assign(b, e);
295#endif
296
297 }
298
299 template<class Token, class Value>
300 static void plus_equal(Token &, const Value &) { }
301
302 // If we are doing an assign, there is no need for the
303 // the clear.
304 //
305 template<class Token>
306 static void clear(Token &) { }
307 };
308
309 template <>
310 struct assign_or_plus_equal<std::input_iterator_tag> {
311 template<class Iterator, class Token>
312 static void assign(Iterator b, Iterator e, Token &t) { }
313 template<class Token, class Value>
314 static void plus_equal(Token &t, const Value &v) {
315 t += v;
316 }
317 template<class Token>
318 static void clear(Token &t) {
319 t = Token();
320 }
321 };
322
323
324 template<class Iterator>
325 struct pointer_iterator_category{
326 typedef std::random_access_iterator_tag type;
327 };
328
329
330 template<class Iterator>
331 struct class_iterator_category{
332 typedef typename Iterator::iterator_category type;
333 };
334
335
336
337 // This portably gets the iterator_tag without partial template specialization
338 template<class Iterator>
339 struct get_iterator_category{
340 typedef typename mpl::if_<is_pointer<Iterator>,
341 pointer_iterator_category<Iterator>,
342 class_iterator_category<Iterator>
343 >::type cat;
344
345 typedef typename cat::type iterator_category;
346 };
347
348
349 } // namespace tokenizer_detail
350
351
352 //===========================================================================
353 // The offset_separator class, which is a model of TokenizerFunction.
354 // Offset breaks a string into tokens based on a range of offsets
355
356 class offset_separator {
357 private:
358
359 std::vector<int> offsets_;
360 unsigned int current_offset_;
361 bool wrap_offsets_;
362 bool return_partial_last_;
363
364 public:
365 template <typename Iter>
366 offset_separator(Iter begin, Iter end, bool wrap_offsets = true,
367 bool return_partial_last = true)
368 : offsets_(begin,end), current_offset_(0),
369 wrap_offsets_(wrap_offsets),
370 return_partial_last_(return_partial_last) { }
371
372 offset_separator()
373 : offsets_(1,1), current_offset_(),
374 wrap_offsets_(true), return_partial_last_(true) { }
375
376 void reset() {
377 current_offset_ = 0;
378 }
379
380 template <typename InputIterator, typename Token>
381 bool operator()(InputIterator& next, InputIterator end, Token& tok)
382 {
383 typedef tokenizer_detail::assign_or_plus_equal<
384 BOOST_DEDUCED_TYPENAME tokenizer_detail::get_iterator_category<
385 InputIterator
386 >::iterator_category
387 > assigner;
388
389 BOOST_ASSERT(!offsets_.empty());
390
391 assigner::clear(tok);
392 InputIterator start(next);
393
394 if (next == end)
395 return false;
396
397 if (current_offset_ == offsets_.size())
398 {
399 if (wrap_offsets_)
400 current_offset_=0;
401 else
402 return false;
403 }
404
405 int c = offsets_[current_offset_];
406 int i = 0;
407 for (; i < c; ++i) {
408 if (next == end)break;
409 assigner::plus_equal(tok,*next++);
410 }
411 assigner::assign(start,next,tok);
412
413 if (!return_partial_last_)
414 if (i < (c-1) )
415 return false;
416
417 ++current_offset_;
418 return true;
419 }
420 };
421
422
423 //===========================================================================
424 // The char_separator class breaks a sequence of characters into
425 // tokens based on the character delimiters (very much like bad old
426 // strtok). A delimiter character can either be kept or dropped. A
427 // kept delimiter shows up as an output token, whereas a dropped
428 // delimiter does not.
429
430 // This class replaces the char_delimiters_separator class. The
431 // constructor for the char_delimiters_separator class was too
432 // confusing and needed to be deprecated. However, because of the
433 // default arguments to the constructor, adding the new constructor
434 // would cause ambiguity, so instead I deprecated the whole class.
435 // The implementation of the class was also simplified considerably.
436
437 enum empty_token_policy { drop_empty_tokens, keep_empty_tokens };
438
439 // The out of the box GCC 2.95 on cygwin does not have a char_traits class.
440 template <typename Char,
441 typename Tr = BOOST_DEDUCED_TYPENAME std::basic_string<Char>::traits_type >
442 class char_separator
443 {
444 typedef tokenizer_detail::traits_extension<Tr> Traits;
445 typedef std::basic_string<Char,Tr> string_type;
446 public:
447 explicit
448 char_separator(const Char* dropped_delims,
449 const Char* kept_delims = 0,
450 empty_token_policy empty_tokens = drop_empty_tokens)
451 : m_dropped_delims(dropped_delims),
452 m_use_ispunct(false),
453 m_use_isspace(false),
454 m_empty_tokens(empty_tokens),
455 m_output_done(false)
456 {
457 // Borland workaround
458 if (kept_delims)
459 m_kept_delims = kept_delims;
460 }
461
462 // use ispunct() for kept delimiters and isspace for dropped.
463 explicit
464 char_separator()
465 : m_use_ispunct(true),
466 m_use_isspace(true),
467 m_empty_tokens(drop_empty_tokens) { }
468
469 void reset() { }
470
471 template <typename InputIterator, typename Token>
472 bool operator()(InputIterator& next, InputIterator end, Token& tok)
473 {
474 typedef tokenizer_detail::assign_or_plus_equal<
475 BOOST_DEDUCED_TYPENAME tokenizer_detail::get_iterator_category<
476 InputIterator
477 >::iterator_category
478 > assigner;
479
480 assigner::clear(tok);
481
482 // skip past all dropped_delims
483 if (m_empty_tokens == drop_empty_tokens)
484 for (; next != end && is_dropped(*next); ++next)
485 { }
486
487 InputIterator start(next);
488
489 if (m_empty_tokens == drop_empty_tokens) {
490
491 if (next == end)
492 return false;
493
494
495 // if we are on a kept_delims move past it and stop
496 if (is_kept(*next)) {
497 assigner::plus_equal(tok,*next);
498 ++next;
499 } else
500 // append all the non delim characters
501 for (; next != end && !is_dropped(*next) && !is_kept(*next); ++next)
502 assigner::plus_equal(tok,*next);
503 }
504 else { // m_empty_tokens == keep_empty_tokens
505
506 // Handle empty token at the end
507 if (next == end)
508 {
509 if (m_output_done == false)
510 {
511 m_output_done = true;
512 assigner::assign(start,next,tok);
513 return true;
514 }
515 else
516 return false;
517 }
518
519 if (is_kept(*next)) {
520 if (m_output_done == false)
521 m_output_done = true;
522 else {
523 assigner::plus_equal(tok,*next);
524 ++next;
525 m_output_done = false;
526 }
527 }
528 else if (m_output_done == false && is_dropped(*next)) {
529 m_output_done = true;
530 }
531 else {
532 if (is_dropped(*next))
533 start=++next;
534 for (; next != end && !is_dropped(*next) && !is_kept(*next); ++next)
535 assigner::plus_equal(tok,*next);
536 m_output_done = true;
537 }
538 }
539 assigner::assign(start,next,tok);
540 return true;
541 }
542
543 private:
544 string_type m_kept_delims;
545 string_type m_dropped_delims;
546 bool m_use_ispunct;
547 bool m_use_isspace;
548 empty_token_policy m_empty_tokens;
549 bool m_output_done;
550
551 bool is_kept(Char E) const
552 {
553 if (m_kept_delims.length())
554 return m_kept_delims.find(E) != string_type::npos;
555 else if (m_use_ispunct) {
556 return Traits::ispunct(E) != 0;
557 } else
558 return false;
559 }
560 bool is_dropped(Char E) const
561 {
562 if (m_dropped_delims.length())
563 return m_dropped_delims.find(E) != string_type::npos;
564 else if (m_use_isspace) {
565 return Traits::isspace(E) != 0;
566 } else
567 return false;
568 }
569 };
570
571 //===========================================================================
572 // The following class is DEPRECATED, use class char_separators instead.
573 //
574 // The char_delimiters_separator class, which is a model of
575 // TokenizerFunction. char_delimiters_separator breaks a string
576 // into tokens based on character delimiters. There are 2 types of
577 // delimiters. returnable delimiters can be returned as
578 // tokens. These are often punctuation. nonreturnable delimiters
579 // cannot be returned as tokens. These are often whitespace
580
581 // The out of the box GCC 2.95 on cygwin does not have a char_traits class.
582 template <class Char,
583 class Tr = BOOST_DEDUCED_TYPENAME std::basic_string<Char>::traits_type >
584 class char_delimiters_separator {
585 private:
586
587 typedef tokenizer_detail::traits_extension<Tr> Traits;
588 typedef std::basic_string<Char,Tr> string_type;
589 string_type returnable_;
590 string_type nonreturnable_;
591 bool return_delims_;
592 bool no_ispunct_;
593 bool no_isspace_;
594
595 bool is_ret(Char E)const
596 {
597 if (returnable_.length())
598 return returnable_.find(E) != string_type::npos;
599 else{
600 if (no_ispunct_) {return false;}
601 else{
602 int r = Traits::ispunct(E);
603 return r != 0;
604 }
605 }
606 }
607 bool is_nonret(Char E)const
608 {
609 if (nonreturnable_.length())
610 return nonreturnable_.find(E) != string_type::npos;
611 else{
612 if (no_isspace_) {return false;}
613 else{
614 int r = Traits::isspace(E);
615 return r != 0;
616 }
617 }
618 }
619
620 public:
621 explicit char_delimiters_separator(bool return_delims = false,
622 const Char* returnable = 0,
623 const Char* nonreturnable = 0)
624 : returnable_(returnable ? returnable : string_type().c_str()),
625 nonreturnable_(nonreturnable ? nonreturnable:string_type().c_str()),
626 return_delims_(return_delims), no_ispunct_(returnable!=0),
627 no_isspace_(nonreturnable!=0) { }
628
629 void reset() { }
630
631 public:
632
633 template <typename InputIterator, typename Token>
634 bool operator()(InputIterator& next, InputIterator end,Token& tok) {
635 tok = Token();
636
637 // skip past all nonreturnable delims
638 // skip past the returnable only if we are not returning delims
639 for (;next!=end && ( is_nonret(*next) || (is_ret(*next)
640 && !return_delims_ ) );++next) { }
641
642 if (next == end) {
643 return false;
644 }
645
646 // if we are to return delims and we are one a returnable one
647 // move past it and stop
648 if (is_ret(*next) && return_delims_) {
649 tok+=*next;
650 ++next;
651 }
652 else
653 // append all the non delim characters
654 for (;next!=end && !is_nonret(*next) && !is_ret(*next);++next)
655 tok+=*next;
656
657
658 return true;
659 }
660 };
661
662
663} //namespace boost
664
665#endif