PageRenderTime 41ms CodeModel.GetById 17ms app.highlight 21ms RepoModel.GetById 1ms app.codeStats 0ms

/Src/Dependencies/Boost/boost/spirit/home/lex/lexer/lexertl/iterator_tokenizer.hpp

http://hadesmem.googlecode.com/
C++ Header | 255 lines | 211 code | 36 blank | 8 comment | 46 complexity | 5c4988e2cf625a8a8d81ef9921451295 MD5 | raw file
  1//  Copyright (c) 2001-2011 Hartmut Kaiser
  2// 
  3//  Distributed under the Boost Software License, Version 1.0. (See accompanying 
  4//  file LICENSE_1_0.txt or copy at http://www.boost.org/LICENSE_1_0.txt)
  5
  6#if !defined(BOOST_SPIRIT_LEXERTL_ITERATOR_TOKENISER_MARCH_22_2007_0859AM)
  7#define BOOST_SPIRIT_LEXERTL_ITERATOR_TOKENISER_MARCH_22_2007_0859AM
  8
  9#if defined(_MSC_VER)
 10#pragma once
 11#endif
 12
 13#include <boost/detail/iterator.hpp>
 14#include <boost/spirit/home/support/detail/lexer/state_machine.hpp>
 15#include <boost/spirit/home/support/detail/lexer/consts.hpp>
 16#include <boost/spirit/home/support/detail/lexer/size_t.hpp>
 17#include <boost/spirit/home/support/detail/lexer/char_traits.hpp>
 18#include <vector>
 19
 20namespace boost { namespace spirit { namespace lex { namespace lexertl
 21{ 
 22    ///////////////////////////////////////////////////////////////////////////
 23    template<typename Iterator>
 24    class basic_iterator_tokeniser
 25    {
 26    public:
 27        typedef std::vector<std::size_t> size_t_vector;
 28        typedef typename boost::detail::iterator_traits<Iterator>::value_type 
 29            char_type;
 30
 31        static std::size_t next (
 32            boost::lexer::basic_state_machine<char_type> const& state_machine_
 33          , std::size_t &dfa_state_, bool& bol_, Iterator &start_token_
 34          , Iterator const& end_, std::size_t& unique_id_)
 35        {
 36            if (start_token_ == end_) 
 37            {
 38                unique_id_ = boost::lexer::npos;
 39                return 0;
 40            }
 41
 42            bool bol = bol_;
 43            boost::lexer::detail::internals const& internals_ =
 44                state_machine_.data();
 45
 46        again:
 47            std::size_t const* lookup_ = &internals_._lookup[dfa_state_]->
 48                front ();
 49            std::size_t dfa_alphabet_ = internals_._dfa_alphabet[dfa_state_];
 50            std::size_t const* dfa_ = &internals_._dfa[dfa_state_]->front ();
 51
 52            std::size_t const* ptr_ = dfa_ + dfa_alphabet_;
 53            Iterator curr_ = start_token_;
 54            bool end_state_ = *ptr_ != 0;
 55            std::size_t id_ = *(ptr_ + boost::lexer::id_index);
 56            std::size_t uid_ = *(ptr_ + boost::lexer::unique_id_index);
 57            std::size_t end_start_state_ = dfa_state_;
 58            bool end_bol_ = bol_;
 59            Iterator end_token_ = start_token_;
 60
 61            while (curr_ != end_)
 62            {
 63                std::size_t const BOL_state_ = ptr_[boost::lexer::bol_index];
 64                std::size_t const EOL_state_ = ptr_[boost::lexer::eol_index];
 65
 66                if (BOL_state_ && bol)
 67                {
 68                    ptr_ = &dfa_[BOL_state_ * dfa_alphabet_];
 69                }
 70                else if (EOL_state_ && *curr_ == '\n')
 71                {
 72                    ptr_ = &dfa_[EOL_state_ * dfa_alphabet_];
 73                }
 74                else
 75                {
 76                    typedef typename 
 77                        boost::detail::iterator_traits<Iterator>::value_type 
 78                    value_type;
 79                    typedef typename 
 80                        boost::lexer::char_traits<value_type>::index_type 
 81                    index_type;
 82
 83                    index_type index = 
 84                        boost::lexer::char_traits<value_type>::call(*curr_++);
 85                    bol = (index == '\n') ? true : false;
 86                    std::size_t const state_ = ptr_[
 87                        lookup_[static_cast<std::size_t>(index)]];
 88
 89                    if (state_ == 0)
 90                    {
 91                        break;
 92                    }
 93
 94                    ptr_ = &dfa_[state_ * dfa_alphabet_];
 95                }
 96
 97                if (*ptr_)
 98                {
 99                    end_state_ = true;
100                    id_ = *(ptr_ + boost::lexer::id_index);
101                    uid_ = *(ptr_ + boost::lexer::unique_id_index);
102                    end_start_state_ = *(ptr_ + boost::lexer::state_index);
103                    end_bol_ = bol;
104                    end_token_ = curr_;
105                }
106            }
107
108            std::size_t const EOL_state_ = ptr_[boost::lexer::eol_index];
109
110            if (EOL_state_ && curr_ == end_)
111            {
112                ptr_ = &dfa_[EOL_state_ * dfa_alphabet_];
113
114                if (*ptr_)
115                {
116                    end_state_ = true;
117                    id_ = *(ptr_ + boost::lexer::id_index);
118                    uid_ = *(ptr_ + boost::lexer::unique_id_index);
119                    end_start_state_ = *(ptr_ + boost::lexer::state_index);
120                    end_bol_ = bol;
121                    end_token_ = curr_;
122                }
123            }
124
125            if (end_state_) {
126                // return longest match
127                dfa_state_ = end_start_state_;
128                start_token_ = end_token_;
129
130                if (id_ == 0)
131                {
132                    bol = end_bol_;
133                    goto again;
134                }
135                else
136                {
137                    bol_ = end_bol_;
138                }
139            }
140            else {
141                bol_ = (*start_token_ == '\n') ? true : false;
142                id_ = boost::lexer::npos;
143                uid_ = boost::lexer::npos;
144            }
145
146            unique_id_ = uid_;
147            return id_;
148        }
149
150        ///////////////////////////////////////////////////////////////////////
151        static std::size_t next (
152            boost::lexer::basic_state_machine<char_type> const& state_machine_
153          , bool& bol_, Iterator &start_token_, Iterator const& end_
154          , std::size_t& unique_id_)
155        {
156            if (start_token_ == end_)
157            {
158                unique_id_ = boost::lexer::npos;
159                return 0;
160            }
161
162            bool bol = bol_;
163            std::size_t const* lookup_ = &state_machine_.data()._lookup[0]->front();
164            std::size_t dfa_alphabet_ = state_machine_.data()._dfa_alphabet[0];
165            std::size_t const* dfa_ = &state_machine_.data()._dfa[0]->front ();
166            std::size_t const* ptr_ = dfa_ + dfa_alphabet_;
167
168            Iterator curr_ = start_token_;
169            bool end_state_ = *ptr_ != 0;
170            std::size_t id_ = *(ptr_ + boost::lexer::id_index);
171            std::size_t uid_ = *(ptr_ + boost::lexer::unique_id_index);
172            bool end_bol_ = bol_;
173            Iterator end_token_ = start_token_;
174
175            while (curr_ != end_)
176            {
177                std::size_t const BOL_state_ = ptr_[boost::lexer::bol_index];
178                std::size_t const EOL_state_ = ptr_[boost::lexer::eol_index];
179
180                if (BOL_state_ && bol)
181                {
182                    ptr_ = &dfa_[BOL_state_ * dfa_alphabet_];
183                }
184                else if (EOL_state_ && *curr_ == '\n')
185                {
186                    ptr_ = &dfa_[EOL_state_ * dfa_alphabet_];
187                }
188                else
189                {
190                    typedef typename 
191                        boost::detail::iterator_traits<Iterator>::value_type 
192                    value_type;
193                    typedef typename 
194                        boost::lexer::char_traits<value_type>::index_type 
195                    index_type;
196
197                    index_type index = 
198                        boost::lexer::char_traits<value_type>::call(*curr_++);
199                    bol = (index == '\n') ? true : false;
200                    std::size_t const state_ = ptr_[
201                        lookup_[static_cast<std::size_t>(index)]];
202
203                    if (state_ == 0)
204                    {
205                        break;
206                    }
207
208                    ptr_ = &dfa_[state_ * dfa_alphabet_];
209                }
210
211                if (*ptr_)
212                {
213                    end_state_ = true;
214                    id_ = *(ptr_ + boost::lexer::id_index);
215                    uid_ = *(ptr_ + boost::lexer::unique_id_index);
216                    end_bol_ = bol;
217                    end_token_ = curr_;
218                }
219            }
220
221            std::size_t const EOL_state_ = ptr_[boost::lexer::eol_index];
222
223            if (EOL_state_ && curr_ == end_)
224            {
225                ptr_ = &dfa_[EOL_state_ * dfa_alphabet_];
226
227                if (*ptr_)
228                {
229                    end_state_ = true;
230                    id_ = *(ptr_ + boost::lexer::id_index);
231                    uid_ = *(ptr_ + boost::lexer::unique_id_index);
232                    end_bol_ = bol;
233                    end_token_ = curr_;
234                }
235            }
236
237            if (end_state_) {
238                // return longest match
239                bol_ = end_bol_;
240                start_token_ = end_token_;
241            }
242            else {
243                bol_ = *start_token_ == '\n';
244                id_ = boost::lexer::npos;
245                uid_ = boost::lexer::npos;
246            }
247
248            unique_id_ = uid_;
249            return id_;
250        }
251    };
252
253}}}}
254
255#endif