/Src/Dependencies/Boost/boost/token_functions.hpp

  1// Boost token_functions.hpp  ------------------------------------------------//
  2
  3// Copyright John R. Bandela 2001. 
  4
  5// Distributed under the Boost Software License, Version 1.0. (See
  6// accompanying file LICENSE_1_0.txt or copy at
  7// http://www.boost.org/LICENSE_1_0.txt)
  8
  9// See http://www.boost.org/libs/tokenizer/ for documentation.
 10
 11// Revision History:
 12// 01 Oct 2004   Joaquin M Lopez Munoz
 13//      Workaround for a problem with string::assign in msvc-stlport
 14// 06 Apr 2004   John Bandela
 15//      Fixed a bug involving using char_delimiter with a true input iterator
 16// 28 Nov 2003   Robert Zeh and John Bandela
 17//      Converted into "fast" functions that avoid using += when
 18//      the supplied iterator isn't an input_iterator; based on
 19//      some work done at Archelon and a version that was checked into
 20//      the boost CVS for a short period of time.
 21// 20 Feb 2002   John Maddock
 22//      Removed using namespace std declarations and added
 23//      workaround for BOOST_NO_STDC_NAMESPACE (the library
 24//      can be safely mixed with regex).
 25// 06 Feb 2002   Jeremy Siek
 26//      Added char_separator.
 27// 02 Feb 2002   Jeremy Siek
 28//      Removed tabs and a little cleanup.
 29
 30
 31#ifndef BOOST_TOKEN_FUNCTIONS_JRB120303_HPP_
 32#define BOOST_TOKEN_FUNCTIONS_JRB120303_HPP_
 33
 34#include <vector>
 35#include <stdexcept>
 36#include <string>
 37#include <cctype>
 38#include <algorithm> // for find_if
 39#include <boost/config.hpp>
 40#include <boost/assert.hpp>
 41#include <boost/detail/workaround.hpp>
 42#include <boost/mpl/if.hpp>
 43#if !defined(BOOST_NO_CWCTYPE)
 44#include <cwctype>
 45#endif
 46
 47//
 48// the following must not be macros if we are to prefix them
 49// with std:: (they shouldn't be macros anyway...)
 50//
 51#ifdef ispunct
 52#  undef ispunct
 53#endif
 54#ifdef iswpunct
 55#  undef iswpunct
 56#endif
 57#ifdef isspace
 58#  undef isspace
 59#endif
 60#ifdef iswspace
 61#  undef iswspace
 62#endif
 63//
 64// fix namespace problems:
 65//
 66#ifdef BOOST_NO_STDC_NAMESPACE
 67namespace std{
 68 using ::ispunct;
 69 using ::isspace;
 70#if !defined(BOOST_NO_CWCTYPE)
 71 using ::iswpunct;
 72 using ::iswspace;
 73#endif
 74}
 75#endif
 76
 77namespace boost{
 78  //===========================================================================
 79  // The escaped_list_separator class. Which is a model of TokenizerFunction
 80  // An escaped list is a super-set of what is commonly known as a comma 
 81  // separated value (csv) list.It is separated into fields by a comma or 
 82  // other character. If the delimiting character is inside quotes, then it is
 83  // counted as a regular character.To allow for embedded quotes in a field,
 84  // there can be escape sequences using the \ much like C. 
 85  // The role of the comma, the quotation mark, and the escape 
 86  // character (backslash \), can be assigned to other characters.
 87
 88  struct escaped_list_error : public std::runtime_error{
 89    escaped_list_error(const std::string& what_arg):std::runtime_error(what_arg) { }
 90  };
 91
 92
 93// The out of the box GCC 2.95 on cygwin does not have a char_traits class.
 94// MSVC does not like the following typename
 95  template <class Char,
 96    class Traits = BOOST_DEDUCED_TYPENAME std::basic_string<Char>::traits_type >
 97  class escaped_list_separator {
 98
 99  private:
100    typedef std::basic_string<Char,Traits> string_type;
101    struct char_eq {
102      Char e_;
103      char_eq(Char e):e_(e) { }
104      bool operator()(Char c) {
105        return Traits::eq(e_,c);
106      }
107    };
108    string_type  escape_;
109    string_type  c_;
110    string_type  quote_;
111    bool last_;
112
113    bool is_escape(Char e) {
114      char_eq f(e);
115      return std::find_if(escape_.begin(),escape_.end(),f)!=escape_.end();
116    }
117    bool is_c(Char e) {
118      char_eq f(e);
119      return std::find_if(c_.begin(),c_.end(),f)!=c_.end();
120    }
121    bool is_quote(Char e) {
122      char_eq f(e);
123      return std::find_if(quote_.begin(),quote_.end(),f)!=quote_.end();
124    }
125    template <typename iterator, typename Token>
126    void do_escape(iterator& next,iterator end,Token& tok) {
127      if (++next == end)
128        throw escaped_list_error(std::string("cannot end with escape"));
129      if (Traits::eq(*next,'n')) {
130        tok+='\n';
131        return;
132      }
133      else if (is_quote(*next)) {
134        tok+=*next;
135        return;
136      }
137      else if (is_c(*next)) {
138        tok+=*next;
139        return;
140      }
141      else if (is_escape(*next)) {
142        tok+=*next;
143        return;
144      }
145      else
146        throw escaped_list_error(std::string("unknown escape sequence"));
147    }
148
149    public:
150
151    explicit escaped_list_separator(Char  e = '\\',
152                                    Char c = ',',Char  q = '\"')
153      : escape_(1,e), c_(1,c), quote_(1,q), last_(false) { }
154
155    escaped_list_separator(string_type e, string_type c, string_type q)
156      : escape_(e), c_(c), quote_(q), last_(false) { }
157
158    void reset() {last_=false;}
159
160    template <typename InputIterator, typename Token>
161    bool operator()(InputIterator& next,InputIterator end,Token& tok) {
162      bool bInQuote = false;
163      tok = Token();
164
165      if (next == end) {
166        if (last_) {
167          last_ = false;
168          return true;
169        }
170        else
171          return false;
172      }
173      last_ = false;
174      for (;next != end;++next) {
175        if (is_escape(*next)) {
176          do_escape(next,end,tok);
177        }
178        else if (is_c(*next)) {
179          if (!bInQuote) {
180            // If we are not in quote, then we are done
181            ++next;
182            // The last character was a c, that means there is
183            // 1 more blank field
184            last_ = true; 
185            return true;
186          }
187          else tok+=*next;
188        }
189        else if (is_quote(*next)) {
190          bInQuote=!bInQuote;
191        }
192        else {
193          tok += *next;
194        }
195      }
196      return true;
197    }
198  };
199
200  //===========================================================================
201  // The classes here are used by offset_separator and char_separator to implement
202  // faster assigning of tokens using assign instead of +=
203
204  namespace tokenizer_detail {
205  //===========================================================================
206  // Tokenizer was broken for wide character separators, at least on Windows, since
207  // CRT functions isspace etc only expect values in [0, 0xFF]. Debug build asserts
208  // if higher values are passed in. The traits extension class should take care of this.
209  // Assuming that the conditional will always get optimized out in the function
210  // implementations, argument types are not a problem since both forms of character classifiers
211  // expect an int.
212   
213#if !defined(BOOST_NO_CWCTYPE)
214  template<typename traits, int N>
215  struct traits_extension_details : public traits {
216    typedef typename traits::char_type char_type;
217    static bool isspace(char_type c)
218    {
219       return std::iswspace(c) != 0;
220    }
221    static bool ispunct(char_type c)
222    {
223       return std::iswpunct(c) != 0;
224    }
225  };
226
227  template<typename traits>
228  struct traits_extension_details<traits, 1> : public traits {
229    typedef typename traits::char_type char_type;
230    static bool isspace(char_type c)
231    {
232       return std::isspace(c) != 0;
233    }
234    static bool ispunct(char_type c)
235    {
236       return std::ispunct(c) != 0;
237    }
238  };
239#endif
240
241    
242  // In case there is no cwctype header, we implement the checks manually.
243  // We make use of the fact that the tested categories should fit in ASCII.
244  template<typename traits>
245  struct traits_extension : public traits {
246    typedef typename traits::char_type char_type;
247    static bool isspace(char_type c)
248    {
249#if !defined(BOOST_NO_CWCTYPE)
250      return traits_extension_details<traits, sizeof(char_type)>::isspace(c);
251#else
252      return static_cast< unsigned >(c) <= 255 && std::isspace(c) != 0;
253#endif
254    }
255
256    static bool ispunct(char_type c)
257    {
258#if !defined(BOOST_NO_CWCTYPE)
259      return traits_extension_details<traits, sizeof(char_type)>::ispunct(c);
260#else
261      return static_cast< unsigned >(c) <= 255 && std::ispunct(c) != 0;
262#endif
263    }
264  };
265
266  // The assign_or_plus_equal struct contains functions that implement
267  // assign, +=, and clearing based on the iterator type.  The
268  // generic case does nothing for plus_equal and clearing, while
269  // passing through the call for assign.
270  //
271  // When an input iterator is being used, the situation is reversed.
272  // The assign method does nothing, plus_equal invokes operator +=,
273  // and the clearing method sets the supplied token to the default
274  // token constructor's result.
275  //
276
277  template<class IteratorTag>
278  struct assign_or_plus_equal {
279    template<class Iterator, class Token>
280    static void assign(Iterator b, Iterator e, Token &t) {
281
282#if BOOST_WORKAROUND(BOOST_MSVC, < 1300) &&\
283    BOOST_WORKAROUND(__SGI_STL_PORT, < 0x500) &&\
284    defined(_STLP_DEBUG) &&\
285    (defined(_STLP_USE_DYNAMIC_LIB) || defined(_DLL))
286    // Problem with string::assign for msvc-stlport in debug mode: the
287    // linker tries to import the templatized version of this memfun,
288    // which is obviously not exported.
289    // See http://www.stlport.com/dcforum/DCForumID6/1763.html for details.
290
291      t = Token();
292      while(b != e) t += *b++;
293#else
294      t.assign(b, e);
295#endif
296
297    }
298
299    template<class Token, class Value>
300    static void plus_equal(Token &, const Value &) { }
301
302    // If we are doing an assign, there is no need for the
303    // the clear.
304    //
305    template<class Token>
306    static void clear(Token &) { }
307  };
308
309  template <>
310  struct assign_or_plus_equal<std::input_iterator_tag> {
311    template<class Iterator, class Token>
312    static void assign(Iterator b, Iterator e, Token &t) { }
313    template<class Token, class Value> 
314    static void plus_equal(Token &t, const Value &v) {
315      t += v;
316    }
317    template<class Token>
318    static void clear(Token &t) {
319      t = Token();
320    }
321  };
322
323
324  template<class Iterator>
325  struct pointer_iterator_category{
326    typedef std::random_access_iterator_tag type;
327  };
328
329
330  template<class Iterator>
331  struct class_iterator_category{
332    typedef typename Iterator::iterator_category type;
333  };
334
335
336
337  // This portably gets the iterator_tag without partial template specialization
338  template<class Iterator>
339    struct get_iterator_category{
340    typedef typename mpl::if_<is_pointer<Iterator>,
341      pointer_iterator_category<Iterator>,
342      class_iterator_category<Iterator>
343    >::type cat;
344
345    typedef typename cat::type iterator_category;
346  };
347
348
349  } // namespace tokenizer_detail
350
351
352  //===========================================================================
353  // The offset_separator class, which is a model of TokenizerFunction.
354  // Offset breaks a string into tokens based on a range of offsets
355
356  class offset_separator {
357  private:
358
359    std::vector<int> offsets_;
360    unsigned int current_offset_;
361    bool wrap_offsets_;
362    bool return_partial_last_;
363
364  public:
365    template <typename Iter>
366    offset_separator(Iter begin, Iter end, bool wrap_offsets = true,
367                     bool return_partial_last = true)
368      : offsets_(begin,end), current_offset_(0),
369        wrap_offsets_(wrap_offsets),
370        return_partial_last_(return_partial_last) { }
371
372    offset_separator()
373      : offsets_(1,1), current_offset_(),
374        wrap_offsets_(true), return_partial_last_(true) { }
375
376    void reset() {
377      current_offset_ = 0;
378    }
379
380    template <typename InputIterator, typename Token>
381    bool operator()(InputIterator& next, InputIterator end, Token& tok)
382    {
383      typedef tokenizer_detail::assign_or_plus_equal<
384        BOOST_DEDUCED_TYPENAME tokenizer_detail::get_iterator_category<
385          InputIterator
386        >::iterator_category
387      > assigner;
388
389      BOOST_ASSERT(!offsets_.empty());
390
391      assigner::clear(tok);
392      InputIterator start(next);
393
394      if (next == end)
395        return false;
396
397      if (current_offset_ == offsets_.size())
398      {
399        if (wrap_offsets_)
400          current_offset_=0;
401        else
402          return false;
403      }
404
405      int c = offsets_[current_offset_];
406      int i = 0;
407      for (; i < c; ++i) {
408        if (next == end)break;
409        assigner::plus_equal(tok,*next++);
410      }
411      assigner::assign(start,next,tok);
412
413      if (!return_partial_last_)
414        if (i < (c-1) )
415          return false;
416
417      ++current_offset_;
418      return true;
419    }
420  };
421
422
423  //===========================================================================
424  // The char_separator class breaks a sequence of characters into
425  // tokens based on the character delimiters (very much like bad old
426  // strtok). A delimiter character can either be kept or dropped. A
427  // kept delimiter shows up as an output token, whereas a dropped
428  // delimiter does not.
429
430  // This class replaces the char_delimiters_separator class. The
431  // constructor for the char_delimiters_separator class was too
432  // confusing and needed to be deprecated. However, because of the
433  // default arguments to the constructor, adding the new constructor
434  // would cause ambiguity, so instead I deprecated the whole class.
435  // The implementation of the class was also simplified considerably.
436
437  enum empty_token_policy { drop_empty_tokens, keep_empty_tokens };
438
439  // The out of the box GCC 2.95 on cygwin does not have a char_traits class.
440  template <typename Char,
441    typename Tr = BOOST_DEDUCED_TYPENAME std::basic_string<Char>::traits_type >
442  class char_separator
443  {
444    typedef tokenizer_detail::traits_extension<Tr> Traits;
445    typedef std::basic_string<Char,Tr> string_type;
446  public:
447    explicit 
448    char_separator(const Char* dropped_delims,
449                   const Char* kept_delims = 0,
450                   empty_token_policy empty_tokens = drop_empty_tokens)
451      : m_dropped_delims(dropped_delims),
452        m_use_ispunct(false),
453        m_use_isspace(false),
454        m_empty_tokens(empty_tokens),
455        m_output_done(false)
456    {
457      // Borland workaround
458      if (kept_delims)
459        m_kept_delims = kept_delims;
460    }
461
462                // use ispunct() for kept delimiters and isspace for dropped.
463    explicit
464    char_separator()
465      : m_use_ispunct(true),
466        m_use_isspace(true),
467        m_empty_tokens(drop_empty_tokens) { }
468
469    void reset() { }
470
471    template <typename InputIterator, typename Token>
472    bool operator()(InputIterator& next, InputIterator end, Token& tok)
473    {
474      typedef tokenizer_detail::assign_or_plus_equal<
475        BOOST_DEDUCED_TYPENAME tokenizer_detail::get_iterator_category<
476          InputIterator
477        >::iterator_category
478      > assigner;
479
480      assigner::clear(tok);
481
482      // skip past all dropped_delims
483      if (m_empty_tokens == drop_empty_tokens)
484        for (; next != end  && is_dropped(*next); ++next)
485          { }
486
487      InputIterator start(next);
488
489      if (m_empty_tokens == drop_empty_tokens) {
490
491        if (next == end)
492          return false;
493
494
495        // if we are on a kept_delims move past it and stop
496        if (is_kept(*next)) {
497          assigner::plus_equal(tok,*next);
498          ++next;
499        } else
500          // append all the non delim characters
501          for (; next != end && !is_dropped(*next) && !is_kept(*next); ++next)
502            assigner::plus_equal(tok,*next);
503      }
504      else { // m_empty_tokens == keep_empty_tokens
505
506        // Handle empty token at the end
507        if (next == end)
508        {
509          if (m_output_done == false)
510          {
511            m_output_done = true;
512            assigner::assign(start,next,tok);
513            return true;
514          } 
515          else
516            return false;
517        }
518
519        if (is_kept(*next)) {
520          if (m_output_done == false)
521            m_output_done = true;
522          else {
523            assigner::plus_equal(tok,*next);
524            ++next;
525            m_output_done = false;
526          }
527        } 
528        else if (m_output_done == false && is_dropped(*next)) {
529          m_output_done = true;
530        } 
531        else {
532          if (is_dropped(*next))
533            start=++next;
534          for (; next != end && !is_dropped(*next) && !is_kept(*next); ++next)
535            assigner::plus_equal(tok,*next);
536          m_output_done = true;
537        }
538      }
539      assigner::assign(start,next,tok);
540      return true;
541    }
542
543  private:
544    string_type m_kept_delims;
545    string_type m_dropped_delims;
546    bool m_use_ispunct;
547    bool m_use_isspace;
548    empty_token_policy m_empty_tokens;
549    bool m_output_done;
550
551    bool is_kept(Char E) const
552    {
553      if (m_kept_delims.length())
554        return m_kept_delims.find(E) != string_type::npos;
555      else if (m_use_ispunct) {
556        return Traits::ispunct(E) != 0;
557      } else
558        return false;
559    }
560    bool is_dropped(Char E) const
561    {
562      if (m_dropped_delims.length())
563        return m_dropped_delims.find(E) != string_type::npos;
564      else if (m_use_isspace) {
565        return Traits::isspace(E) != 0;
566      } else
567        return false;
568    }
569  };
570
571  //===========================================================================
572  // The following class is DEPRECATED, use class char_separators instead.
573  //
574  // The char_delimiters_separator class, which is a model of
575  // TokenizerFunction.  char_delimiters_separator breaks a string
576  // into tokens based on character delimiters. There are 2 types of
577  // delimiters. returnable delimiters can be returned as
578  // tokens. These are often punctuation. nonreturnable delimiters
579  // cannot be returned as tokens. These are often whitespace
580
581  // The out of the box GCC 2.95 on cygwin does not have a char_traits class.
582  template <class Char,
583    class Tr = BOOST_DEDUCED_TYPENAME std::basic_string<Char>::traits_type >
584  class char_delimiters_separator {
585  private:
586
587    typedef tokenizer_detail::traits_extension<Tr> Traits;
588    typedef std::basic_string<Char,Tr> string_type;
589    string_type returnable_;
590    string_type nonreturnable_;
591    bool return_delims_;
592    bool no_ispunct_;
593    bool no_isspace_;
594
595    bool is_ret(Char E)const
596    {
597      if (returnable_.length())
598        return  returnable_.find(E) != string_type::npos;
599      else{
600        if (no_ispunct_) {return false;}
601        else{
602          int r = Traits::ispunct(E);
603          return r != 0;
604        }
605      }
606    }
607    bool is_nonret(Char E)const
608    {
609      if (nonreturnable_.length())
610        return  nonreturnable_.find(E) != string_type::npos;
611      else{
612        if (no_isspace_) {return false;}
613        else{
614          int r = Traits::isspace(E);
615          return r != 0;
616        }
617      }
618    }
619
620  public:
621    explicit char_delimiters_separator(bool return_delims = false, 
622                                       const Char* returnable = 0,
623                                       const Char* nonreturnable = 0)
624      : returnable_(returnable ? returnable : string_type().c_str()),
625        nonreturnable_(nonreturnable ? nonreturnable:string_type().c_str()),
626        return_delims_(return_delims), no_ispunct_(returnable!=0),
627        no_isspace_(nonreturnable!=0) { }
628
629    void reset() { }
630
631  public:
632
633     template <typename InputIterator, typename Token>
634     bool operator()(InputIterator& next, InputIterator end,Token& tok) {
635     tok = Token();
636
637     // skip past all nonreturnable delims
638     // skip past the returnable only if we are not returning delims
639     for (;next!=end && ( is_nonret(*next) || (is_ret(*next) 
640       && !return_delims_ ) );++next) { }
641
642     if (next == end) {
643       return false;
644     }
645
646     // if we are to return delims and we are one a returnable one
647     // move past it and stop
648     if (is_ret(*next) && return_delims_) {
649       tok+=*next;
650       ++next;
651     }
652     else
653       // append all the non delim characters
654       for (;next!=end && !is_nonret(*next) && !is_ret(*next);++next)
655         tok+=*next;
656
657
658     return true;
659   }
660  };
661
662
663} //namespace boost
664
665#endif