/Src/Dependencies/Boost/boost/spirit/home/lex/lexer/lexertl/generate_static.hpp
C++ Header | 961 lines | 823 code | 110 blank | 28 comment | 131 complexity | 96ab24727985ba25cb0110dbed6dae16 MD5 | raw file
1// Copyright (c) 2008-2009 Ben Hanson 2// Copyright (c) 2008-2011 Hartmut Kaiser 3// 4// Distributed under the Boost Software License, Version 1.0. (See accompanying 5// file LICENSE_1_0.txt or copy at http://www.boost.org/LICENSE_1_0.txt) 6 7#if !defined(BOOST_SPIRIT_LEX_LEXERTL_GENERATE_CPP_FEB_10_2008_0855PM) 8#define BOOST_SPIRIT_LEX_LEXERTL_GENERATE_CPP_FEB_10_2008_0855PM 9 10#if defined(_MSC_VER) 11#pragma once 12#endif 13 14#include <boost/spirit/home/support/detail/lexer/char_traits.hpp> 15#include <boost/spirit/home/support/detail/lexer/consts.hpp> 16#include <boost/spirit/home/support/detail/lexer/rules.hpp> 17#include <boost/spirit/home/support/detail/lexer/size_t.hpp> 18#include <boost/spirit/home/support/detail/lexer/state_machine.hpp> 19#include <boost/spirit/home/support/detail/lexer/debug.hpp> 20#include <boost/spirit/home/lex/lexer/lexertl/static_version.hpp> 21#include <boost/algorithm/string.hpp> 22#include <boost/lexical_cast.hpp> 23 24/////////////////////////////////////////////////////////////////////////////// 25namespace boost { namespace spirit { namespace lex { namespace lexertl 26{ 27 namespace detail 28 { 29 30 inline bool 31 generate_delimiter(std::ostream &os_) 32 { 33 os_ << std::string(80, '/') << "\n"; 34 return os_.good(); 35 } 36 37 /////////////////////////////////////////////////////////////////////////// 38 // Generate a table of the names of the used lexer states, which is a bit 39 // tricky, because the table stored with the rules is sorted based on the 40 // names, but we need it sorted using the state ids. 41 template <typename Char> 42 inline bool 43 generate_cpp_state_info (boost::lexer::basic_rules<Char> const& rules_ 44 , std::ostream &os_, char const* name_suffix) 45 { 46 // we need to re-sort the state names in ascending order of the state 47 // ids, filling possible gaps in between later 48 typedef typename 49 boost::lexer::basic_rules<Char>::string_size_t_map::const_iterator 50 state_iterator; 51 typedef std::map<std::size_t, char const*> reverse_state_map_type; 52 53 reverse_state_map_type reverse_state_map; 54 state_iterator send = rules_.statemap().end(); 55 for (state_iterator sit = rules_.statemap().begin(); sit != send; ++sit) 56 { 57 typedef typename reverse_state_map_type::value_type value_type; 58 reverse_state_map.insert(value_type((*sit).second, (*sit).first.c_str())); 59 } 60 61 generate_delimiter(os_); 62 os_ << "// this table defines the names of the lexer states\n"; 63 os_ << "char const* const lexer_state_names" 64 << (name_suffix[0] ? "_" : "") << name_suffix 65 << "[" << rules_.statemap().size() << "] = \n{\n"; 66 67 typedef typename reverse_state_map_type::iterator iterator; 68 iterator rend = reverse_state_map.end(); 69 std::size_t last_id = 0; 70 for (iterator rit = reverse_state_map.begin(); rit != rend; ++last_id) 71 { 72 for (/**/; last_id < (*rit).first; ++last_id) 73 { 74 os_ << " 0, // \"<undefined state>\"\n"; 75 } 76 os_ << " \"" << (*rit).second << "\""; 77 if (++rit != rend) 78 os_ << ",\n"; 79 else 80 os_ << "\n"; // don't generate the final comma 81 } 82 os_ << "};\n\n"; 83 84 generate_delimiter(os_); 85 os_ << "// this variable defines the number of lexer states\n"; 86 os_ << "std::size_t const lexer_state_count" 87 << (name_suffix[0] ? "_" : "") << name_suffix 88 << " = " << rules_.statemap().size() << ";\n\n"; 89 return os_.good(); 90 } 91 92 inline bool 93 generate_cpp_state_table (std::ostream &os_, char const* name_suffix 94 , bool bol, bool eol) 95 { 96 std::string suffix(name_suffix[0] ? "_" : ""); 97 suffix += name_suffix; 98 99 generate_delimiter(os_); 100 os_ << "// this defines a generic accessors for the information above\n"; 101 os_ << "struct lexer" << suffix << "\n{\n"; 102 os_ << " // version number and feature-set of compatible static lexer engine\n"; 103 os_ << " enum\n"; 104 os_ << " {\n static_version = " 105 << boost::lexical_cast<std::string>(SPIRIT_STATIC_LEXER_VERSION) << ",\n"; 106 os_ << " supports_bol = " << std::boolalpha << bol << ",\n"; 107 os_ << " supports_eol = " << std::boolalpha << eol << "\n"; 108 os_ << " };\n\n"; 109 os_ << " // return the number of lexer states\n"; 110 os_ << " static std::size_t state_count()\n"; 111 os_ << " {\n return lexer_state_count" << suffix << "; \n }\n\n"; 112 os_ << " // return the name of the lexer state as given by 'idx'\n"; 113 os_ << " static char const* state_name(std::size_t idx)\n"; 114 os_ << " {\n return lexer_state_names" << suffix << "[idx]; \n }\n\n"; 115 os_ << " // return the next matched token\n"; 116 os_ << " template<typename Iterator>\n"; 117 os_ << " static std::size_t next(std::size_t &start_state_, bool& bol_\n"; 118 os_ << " , Iterator &start_token_, Iterator const& end_, std::size_t& unique_id_)\n"; 119 os_ << " {\n return next_token" << suffix 120 << "(start_state_, bol_, start_token_, end_, unique_id_);\n }\n"; 121 os_ << "};\n\n"; 122 return os_.good(); 123 } 124 125 /////////////////////////////////////////////////////////////////////////// 126 // generate function body based on traversing the DFA tables 127 template <typename Char> 128 bool generate_function_body_dfa(std::ostream & os_ 129 , boost::lexer::basic_state_machine<Char> const &sm_) 130 { 131 std::size_t const dfas_ = sm_.data()._dfa->size(); 132 std::size_t const lookups_ = sm_.data()._lookup->front()->size(); 133 134 os_ << " enum {end_state_index, id_index, unique_id_index, " 135 "state_index, bol_index,\n"; 136 os_ << " eol_index, dead_state_index, dfa_offset};\n\n"; 137 os_ << " static std::size_t const npos = " 138 "static_cast<std::size_t>(~0);\n"; 139 140 if (dfas_ > 1) 141 { 142 for (std::size_t state_ = 0; state_ < dfas_; ++state_) 143 { 144 std::size_t i_ = 0; 145 std::size_t j_ = 1; 146 std::size_t count_ = lookups_ / 8; 147 std::size_t const* lookup_ = &sm_.data()._lookup[state_]->front(); 148 std::size_t const* dfa_ = &sm_.data()._dfa[state_]->front(); 149 150 os_ << " static std::size_t const lookup" << state_ 151 << "_[" << lookups_ << "] = {\n "; 152 for (/**/; i_ < count_; ++i_) 153 { 154 std::size_t const index_ = i_ * 8; 155 os_ << lookup_[index_]; 156 for (/**/; j_ < 8; ++j_) 157 { 158 os_ << ", " << lookup_[index_ + j_]; 159 } 160 if (i_ < count_ - 1) 161 { 162 os_ << ",\n "; 163 } 164 j_ = 1; 165 } 166 os_ << " };\n"; 167 168 count_ = sm_.data()._dfa[state_]->size (); 169 os_ << " static const std::size_t dfa" << state_ << "_[" 170 << count_ << "] = {\n "; 171 count_ /= 8; 172 for (i_ = 0; i_ < count_; ++i_) 173 { 174 std::size_t const index_ = i_ * 8; 175 os_ << dfa_[index_]; 176 for (j_ = 1; j_ < 8; ++j_) 177 { 178 os_ << ", " << dfa_[index_ + j_]; 179 } 180 if (i_ < count_ - 1) 181 { 182 os_ << ",\n "; 183 } 184 } 185 186 std::size_t const mod_ = sm_.data()._dfa[state_]->size () % 8; 187 if (mod_) 188 { 189 std::size_t const index_ = count_ * 8; 190 if (count_) 191 { 192 os_ << ",\n "; 193 } 194 os_ << dfa_[index_]; 195 for (j_ = 1; j_ < mod_; ++j_) 196 { 197 os_ << ", " << dfa_[index_ + j_]; 198 } 199 } 200 os_ << " };\n"; 201 } 202 203 std::size_t count_ = sm_.data()._dfa_alphabet.size(); 204 std::size_t i_ = 1; 205 206 os_ << " static std::size_t const* lookup_arr_[" << count_ 207 << "] = { lookup0_"; 208 for (i_ = 1; i_ < count_; ++i_) 209 { 210 os_ << ", " << "lookup" << i_ << "_"; 211 } 212 os_ << " };\n"; 213 214 os_ << " static std::size_t const dfa_alphabet_arr_[" 215 << count_ << "] = { "; 216 os_ << sm_.data()._dfa_alphabet.front (); 217 for (i_ = 1; i_ < count_; ++i_) 218 { 219 os_ << ", " << sm_.data()._dfa_alphabet[i_]; 220 } 221 os_ << " };\n"; 222 223 os_ << " static std::size_t const* dfa_arr_[" << count_ 224 << "] = { "; 225 os_ << "dfa0_"; 226 for (i_ = 1; i_ < count_; ++i_) 227 { 228 os_ << ", " << "dfa" << i_ << "_"; 229 } 230 os_ << " };\n"; 231 } 232 else 233 { 234 std::size_t const* lookup_ = &sm_.data()._lookup[0]->front(); 235 std::size_t const* dfa_ = &sm_.data()._dfa[0]->front(); 236 std::size_t i_ = 0; 237 std::size_t j_ = 1; 238 std::size_t count_ = lookups_ / 8; 239 240 os_ << " static std::size_t const lookup_["; 241 os_ << sm_.data()._lookup[0]->size() << "] = {\n "; 242 for (/**/; i_ < count_; ++i_) 243 { 244 const std::size_t index_ = i_ * 8; 245 os_ << lookup_[index_]; 246 for (/**/; j_ < 8; ++j_) 247 { 248 os_ << ", " << lookup_[index_ + j_]; 249 } 250 if (i_ < count_ - 1) 251 { 252 os_ << ",\n "; 253 } 254 j_ = 1; 255 } 256 os_ << " };\n"; 257 258 os_ << " static std::size_t const dfa_alphabet_ = " 259 << sm_.data()._dfa_alphabet.front () << ";\n"; 260 os_ << " static std::size_t const dfa_[" 261 << sm_.data()._dfa[0]->size () << "] = {\n "; 262 count_ = sm_.data()._dfa[0]->size () / 8; 263 for (i_ = 0; i_ < count_; ++i_) 264 { 265 const std::size_t index_ = i_ * 8; 266 os_ << dfa_[index_]; 267 for (j_ = 1; j_ < 8; ++j_) 268 { 269 os_ << ", " << dfa_[index_ + j_]; 270 } 271 if (i_ < count_ - 1) 272 { 273 os_ << ",\n "; 274 } 275 } 276 277 const std::size_t mod_ = sm_.data()._dfa[0]->size () % 8; 278 if (mod_) 279 { 280 const std::size_t index_ = count_ * 8; 281 if (count_) 282 { 283 os_ << ",\n "; 284 } 285 os_ << dfa_[index_]; 286 for (j_ = 1; j_ < mod_; ++j_) 287 { 288 os_ << ", " << dfa_[index_ + j_]; 289 } 290 } 291 os_ << " };\n"; 292 } 293 294 os_ << "\n if (start_token_ == end_)\n"; 295 os_ << " {\n"; 296 os_ << " unique_id_ = npos;\n"; 297 os_ << " return 0;\n"; 298 os_ << " }\n\n"; 299 if (sm_.data()._seen_BOL_assertion) 300 { 301 os_ << " bool bol = bol_;\n\n"; 302 } 303 304 if (dfas_ > 1) 305 { 306 os_ << "again:\n"; 307 os_ << " std::size_t const* lookup_ = lookup_arr_[start_state_];\n"; 308 os_ << " std::size_t dfa_alphabet_ = dfa_alphabet_arr_[start_state_];\n"; 309 os_ << " std::size_t const*dfa_ = dfa_arr_[start_state_];\n"; 310 } 311 312 os_ << " std::size_t const* ptr_ = dfa_ + dfa_alphabet_;\n"; 313 os_ << " Iterator curr_ = start_token_;\n"; 314 os_ << " bool end_state_ = *ptr_ != 0;\n"; 315 os_ << " std::size_t id_ = *(ptr_ + id_index);\n"; 316 os_ << " std::size_t uid_ = *(ptr_ + unique_id_index);\n"; 317 if (dfas_ > 1) 318 { 319 os_ << " std::size_t end_start_state_ = start_state_;\n"; 320 } 321 if (sm_.data()._seen_BOL_assertion) 322 { 323 os_ << " bool end_bol_ = bol_;\n"; 324 } 325 os_ << " Iterator end_token_ = start_token_;\n\n"; 326 327 os_ << " while (curr_ != end_)\n"; 328 os_ << " {\n"; 329 330 if (sm_.data()._seen_BOL_assertion) 331 { 332 os_ << " std::size_t const BOL_state_ = ptr_[bol_index];\n\n"; 333 } 334 335 if (sm_.data()._seen_EOL_assertion) 336 { 337 os_ << " std::size_t const EOL_state_ = ptr_[eol_index];\n\n"; 338 } 339 340 if (sm_.data()._seen_BOL_assertion && sm_.data()._seen_EOL_assertion) 341 { 342 os_ << " if (BOL_state_ && bol)\n"; 343 os_ << " {\n"; 344 os_ << " ptr_ = &dfa_[BOL_state_ * dfa_alphabet_];\n"; 345 os_ << " }\n"; 346 os_ << " else if (EOL_state_ && *curr_ == '\\n')\n"; 347 os_ << " {\n"; 348 os_ << " ptr_ = &dfa_[EOL_state_ * dfa_alphabet_];\n"; 349 os_ << " }\n"; 350 os_ << " else\n"; 351 os_ << " {\n"; 352 if (lookups_ == 256) 353 { 354 os_ << " unsigned char index = \n"; 355 os_ << " static_cast<unsigned char>(*curr_++);\n"; 356 } 357 else 358 { 359 os_ << " std::size_t index = *curr_++\n"; 360 } 361 os_ << " bol = (index == '\n') ? true : false;\n"; 362 os_ << " std::size_t const state_ = ptr_[\n"; 363 os_ << " lookup_[static_cast<std::size_t>(index)]];\n"; 364 365 os_ << '\n'; 366 os_ << " if (state_ == 0) break;\n"; 367 os_ << '\n'; 368 os_ << " ptr_ = &dfa_[state_ * dfa_alphabet_];\n"; 369 os_ << " }\n\n"; 370 } 371 else if (sm_.data()._seen_BOL_assertion) 372 { 373 os_ << " if (BOL_state_ && bol)\n"; 374 os_ << " {\n"; 375 os_ << " ptr_ = &dfa_[BOL_state_ * dfa_alphabet_];\n"; 376 os_ << " }\n"; 377 os_ << " else\n"; 378 os_ << " {\n"; 379 if (lookups_ == 256) 380 { 381 os_ << " unsigned char index = \n"; 382 os_ << " static_cast<unsigned char>(*curr_++);\n"; 383 } 384 else 385 { 386 os_ << " std::size_t index = *curr_++\n"; 387 } 388 os_ << " bol = (index == '\n') ? true : false;\n"; 389 os_ << " std::size_t const state_ = ptr_[\n"; 390 os_ << " lookup_[static_cast<std::size_t>(index)]];\n"; 391 392 os_ << '\n'; 393 os_ << " if (state_ == 0) break;\n"; 394 os_ << '\n'; 395 os_ << " ptr_ = &dfa_[state_ * dfa_alphabet_];\n"; 396 os_ << " }\n\n"; 397 } 398 else if (sm_.data()._seen_EOL_assertion) 399 { 400 os_ << " if (EOL_state_ && *curr_ == '\\n')\n"; 401 os_ << " {\n"; 402 os_ << " ptr_ = &dfa_[EOL_state_ * dfa_alphabet_];\n"; 403 os_ << " }\n"; 404 os_ << " else\n"; 405 os_ << " {\n"; 406 if (lookups_ == 256) 407 { 408 os_ << " unsigned char index = \n"; 409 os_ << " static_cast<unsigned char>(*curr_++);\n"; 410 } 411 else 412 { 413 os_ << " std::size_t index = *curr_++\n"; 414 } 415 os_ << " bol = (index == '\n') ? true : false;\n"; 416 os_ << " std::size_t const state_ = ptr_[\n"; 417 os_ << " lookup_[static_cast<std::size_t>(index)]];\n"; 418 419 os_ << '\n'; 420 os_ << " if (state_ == 0) break;\n"; 421 os_ << '\n'; 422 os_ << " ptr_ = &dfa_[state_ * dfa_alphabet_];\n"; 423 os_ << " }\n\n"; 424 } 425 else 426 { 427 os_ << " std::size_t const state_ =\n"; 428 429 if (lookups_ == 256) 430 { 431 os_ << " ptr_[lookup_[" 432 "static_cast<unsigned char>(*curr_++)]];\n"; 433 } 434 else 435 { 436 os_ << " ptr_[lookup_[*curr_++]];\n"; 437 } 438 439 os_ << '\n'; 440 os_ << " if (state_ == 0) break;\n"; 441 os_ << '\n'; 442 os_ << " ptr_ = &dfa_[state_ * dfa_alphabet_];\n\n"; 443 } 444 445 os_ << " if (*ptr_)\n"; 446 os_ << " {\n"; 447 os_ << " end_state_ = true;\n"; 448 os_ << " id_ = *(ptr_ + id_index);\n"; 449 os_ << " uid_ = *(ptr_ + unique_id_index);\n"; 450 if (dfas_ > 1) 451 { 452 os_ << " end_start_state_ = *(ptr_ + state_index);\n"; 453 } 454 if (sm_.data()._seen_BOL_assertion) 455 { 456 os_ << " end_bol_ = bol;\n"; 457 } 458 os_ << " end_token_ = curr_;\n"; 459 os_ << " }\n"; 460 os_ << " }\n\n"; 461 462 if (sm_.data()._seen_EOL_assertion) 463 { 464 os_ << " std::size_t const EOL_state_ = ptr_[eol_index];\n\n"; 465 466 os_ << " if (EOL_state_ && curr_ == end_)\n"; 467 os_ << " {\n"; 468 os_ << " ptr_ = &dfa_[EOL_state_ * dfa_alphabet_];\n\n"; 469 470 os_ << " if (*ptr_)\n"; 471 os_ << " {\n"; 472 os_ << " end_state_ = true;\n"; 473 os_ << " id_ = *(ptr_ + id_index);\n"; 474 os_ << " uid_ = *(ptr_ + unique_id_index);\n"; 475 if (dfas_ > 1) 476 { 477 os_ << " end_start_state_ = *(ptr_ + state_index);\n"; 478 } 479 if (sm_.data()._seen_BOL_assertion) 480 { 481 os_ << " end_bol_ = bol;\n"; 482 } 483 os_ << " end_token_ = curr_;\n"; 484 os_ << " }\n"; 485 os_ << " }\n\n"; 486 } 487 488 os_ << " if (end_state_)\n"; 489 os_ << " {\n"; 490 os_ << " // return longest match\n"; 491 os_ << " start_token_ = end_token_;\n"; 492 493 if (dfas_ > 1) 494 { 495 os_ << " start_state_ = end_start_state_;\n"; 496 os_ << " if (id_ == 0)\n"; 497 os_ << " {\n"; 498 if (sm_.data()._seen_BOL_assertion) 499 { 500 os_ << " bol = end_bol_;\n"; 501 } 502 os_ << " goto again;\n"; 503 os_ << " }\n"; 504 if (sm_.data()._seen_BOL_assertion) 505 { 506 os_ << " else\n"; 507 os_ << " {\n"; 508 os_ << " bol_ = end_bol_;\n"; 509 os_ << " }\n"; 510 } 511 } 512 else if (sm_.data()._seen_BOL_assertion) 513 { 514 os_ << " bol_ = end_bol_;\n"; 515 } 516 517 os_ << " }\n"; 518 os_ << " else\n"; 519 os_ << " {\n"; 520 521 if (sm_.data()._seen_BOL_assertion) 522 { 523 os_ << " bol_ = (*start_token_ == '\n') ? true : false;\n"; 524 } 525 526 os_ << " id_ = npos;\n"; 527 os_ << " uid_ = npos;\n"; 528 os_ << " }\n\n"; 529 530 os_ << " unique_id_ = uid_;\n"; 531 os_ << " return id_;\n"; 532 return os_.good(); 533 } 534 535 /////////////////////////////////////////////////////////////////////////// 536 template <typename Char> 537 inline std::string get_charlit(Char ch) 538 { 539 std::basic_string<Char> result; 540 boost::lexer::basic_string_token<Char>::escape_char (ch, result); 541 return result; 542 } 543 544 // check whether state0_0 is referenced from any of the other states 545 template <typename Char> 546 bool need_label0_0(boost::lexer::basic_state_machine<Char> const &sm_) 547 { 548 typedef typename boost::lexer::basic_state_machine<Char>::iterator 549 iterator_type; 550 iterator_type iter_ = sm_.begin(); 551 std::size_t const states_ = iter_->states; 552 553 for (std::size_t state_ = 0; state_ < states_; ++state_) 554 { 555 if (0 == iter_->bol_index || 0 == iter_->eol_index) 556 { 557 return true; 558 } 559 560 std::size_t const transitions_ = iter_->transitions; 561 for (std::size_t t_ = 0; t_ < transitions_; ++t_) 562 { 563 if (0 == iter_->goto_state) 564 { 565 return true; 566 } 567 ++iter_; 568 } 569 if (transitions_ == 0) ++iter_; 570 } 571 return false; 572 } 573 574 /////////////////////////////////////////////////////////////////////////// 575 template <typename Char> 576 bool generate_function_body_switch(std::ostream & os_ 577 , boost::lexer::basic_state_machine<Char> const &sm_) 578 { 579 typedef typename boost::lexer::basic_state_machine<Char>::iterator 580 iterator_type; 581 582 std::size_t const lookups_ = sm_.data()._lookup->front ()->size (); 583 iterator_type iter_ = sm_.begin(); 584 iterator_type labeliter_ = iter_; 585 iterator_type end_ = sm_.end(); 586 std::size_t const dfas_ = sm_.data()._dfa->size (); 587 588 os_ << " static std::size_t const npos = " 589 "static_cast<std::size_t>(~0);\n"; 590 591 os_ << "\n if (start_token_ == end_)\n"; 592 os_ << " {\n"; 593 os_ << " unique_id_ = npos;\n"; 594 os_ << " return 0;\n"; 595 os_ << " }\n\n"; 596 597 if (sm_.data()._seen_BOL_assertion) 598 { 599 os_ << " bool bol = bol_;\n"; 600 } 601 602 if (dfas_ > 1) 603 { 604 os_ << "again:\n"; 605 } 606 607 os_ << " Iterator curr_ = start_token_;\n"; 608 os_ << " bool end_state_ = false;\n"; 609 os_ << " std::size_t id_ = npos;\n"; 610 os_ << " std::size_t uid_ = npos;\n"; 611 612 if (dfas_ > 1) 613 { 614 os_ << " std::size_t end_start_state_ = start_state_;\n"; 615 } 616 617 if (sm_.data()._seen_BOL_assertion) 618 { 619 os_ << " bool end_bol_ = bol_;\n"; 620 } 621 622 os_ << " Iterator end_token_ = start_token_;\n"; 623 os_ << '\n'; 624 625 os_ << " " << ((lookups_ == 256) ? "char" : "wchar_t") 626 << " ch_ = 0;\n\n"; 627 628 if (dfas_ > 1) 629 { 630 os_ << " switch (start_state_)\n"; 631 os_ << " {\n"; 632 633 for (std::size_t i_ = 0; i_ < dfas_; ++i_) 634 { 635 os_ << " case " << i_ << ":\n"; 636 os_ << " goto state" << i_ << "_0;\n"; 637 os_ << " break;\n"; 638 } 639 640 os_ << " default:\n"; 641 os_ << " goto end;\n"; 642 os_ << " break;\n"; 643 os_ << " }\n"; 644 } 645 646 bool need_state0_0_label = need_label0_0(sm_); 647 648 for (std::size_t dfa_ = 0; dfa_ < dfas_; ++dfa_) 649 { 650 std::size_t const states_ = iter_->states; 651 for (std::size_t state_ = 0; state_ < states_; ++state_) 652 { 653 std::size_t const transitions_ = iter_->transitions; 654 std::size_t t_ = 0; 655 656 if (dfas_ > 1 || dfa_ != 0 || state_ != 0 || need_state0_0_label) 657 { 658 os_ << "\nstate" << dfa_ << '_' << state_ << ":\n"; 659 } 660 661 if (iter_->end_state) 662 { 663 os_ << " end_state_ = true;\n"; 664 os_ << " id_ = " << iter_->id << ";\n"; 665 os_ << " uid_ = " << iter_->unique_id << ";\n"; 666 os_ << " end_token_ = curr_;\n"; 667 668 if (dfas_ > 1) 669 { 670 os_ << " end_start_state_ = " << iter_->goto_dfa << 671 ";\n"; 672 } 673 674 if (sm_.data()._seen_BOL_assertion) 675 { 676 os_ << " end_bol_ = bol;\n"; 677 } 678 679 if (transitions_) os_ << '\n'; 680 } 681 682 if (t_ < transitions_ || 683 iter_->bol_index != boost::lexer::npos || 684 iter_->eol_index != boost::lexer::npos) 685 { 686 os_ << " if (curr_ == end_) goto end;\n"; 687 os_ << " ch_ = *curr_;\n"; 688 if (iter_->bol_index != boost::lexer::npos) 689 { 690 os_ << "\n if (bol) goto state" << dfa_ << '_' 691 << iter_->bol_index << ";\n"; 692 } 693 if (iter_->eol_index != boost::lexer::npos) 694 { 695 os_ << "\n if (ch_ == '\n') goto state" << dfa_ 696 << '_' << iter_->eol_index << ";\n"; 697 } 698 os_ << " ++curr_;\n"; 699 } 700 701 for (/**/; t_ < transitions_; ++t_) 702 { 703 char const *ptr_ = iter_->token._charset.c_str(); 704 char const *end_ = ptr_ + iter_->token._charset.size(); 705 char start_char_ = 0; 706 char curr_char_ = 0; 707 bool range_ = false; 708 bool first_char_ = true; 709 710 os_ << "\n if ("; 711 712 while (ptr_ != end_) 713 { 714 curr_char_ = *ptr_++; 715 716 if (*ptr_ == curr_char_ + 1) 717 { 718 if (!range_) 719 { 720 start_char_ = curr_char_; 721 } 722 range_ = true; 723 } 724 else 725 { 726 if (!first_char_) 727 { 728 os_ << ((iter_->token._negated) ? " && " : " || "); 729 } 730 else 731 { 732 first_char_ = false; 733 } 734 if (range_) 735 { 736 if (iter_->token._negated) 737 { 738 os_ << "!"; 739 } 740 os_ << "(ch_ >= '" << get_charlit(start_char_) 741 << "' && ch_ <= '" 742 << get_charlit(curr_char_) << "')"; 743 range_ = false; 744 } 745 else 746 { 747 os_ << "ch_ " 748 << ((iter_->token._negated) ? "!=" : "==") 749 << " '" << get_charlit(curr_char_) << "'"; 750 } 751 } 752 } 753 754 os_ << ") goto state" << dfa_ << '_' << iter_->goto_state 755 << ";\n"; 756 ++iter_; 757 } 758 759 if (!(dfa_ == dfas_ - 1 && state_ == states_ - 1)) 760 { 761 os_ << " goto end;\n"; 762 } 763 764 if (transitions_ == 0) ++iter_; 765 } 766 } 767 768 os_ << "\nend:\n"; 769 os_ << " if (end_state_)\n"; 770 os_ << " {\n"; 771 os_ << " // return longest match\n"; 772 os_ << " start_token_ = end_token_;\n"; 773 774 if (dfas_ > 1) 775 { 776 os_ << " start_state_ = end_start_state_;\n"; 777 os_ << "\n if (id_ == 0)\n"; 778 os_ << " {\n"; 779 780 if (sm_.data()._seen_BOL_assertion) 781 { 782 os_ << " bol = end_bol_;\n"; 783 } 784 785 os_ << " goto again;\n"; 786 os_ << " }\n"; 787 788 if (sm_.data()._seen_BOL_assertion) 789 { 790 os_ << " else\n"; 791 os_ << " {\n"; 792 os_ << " bol_ = end_bol_;\n"; 793 os_ << " }\n"; 794 } 795 } 796 else if (sm_.data()._seen_BOL_assertion) 797 { 798 os_ << " bol_ = end_bol_;\n"; 799 } 800 801 os_ << " }\n"; 802 os_ << " else\n"; 803 os_ << " {\n"; 804 805 if (sm_.data()._seen_BOL_assertion) 806 { 807 os_ << " bol_ = (*start_token_ == '\\n') ? true : false;\n"; 808 } 809 os_ << " id_ = npos;\n"; 810 os_ << " uid_ = npos;\n"; 811 os_ << " }\n\n"; 812 813 os_ << " unique_id_ = uid_;\n"; 814 os_ << " return id_;\n"; 815 return os_.good(); 816 } 817 818 /////////////////////////////////////////////////////////////////////////// 819 // Generate a tokenizer for the given state machine. 820 template <typename Char, typename F> 821 inline bool 822 generate_cpp (boost::lexer::basic_state_machine<Char> const& sm_ 823 , boost::lexer::basic_rules<Char> const& rules_ 824 , std::ostream &os_, char const* name_suffix, F generate_function_body) 825 { 826 if (sm_.data()._lookup->empty()) 827 return false; 828 829 std::size_t const dfas_ = sm_.data()._dfa->size(); 830// std::size_t const lookups_ = sm_.data()._lookup->front()->size(); 831 832 os_ << "// Copyright (c) 2008-2009 Ben Hanson\n"; 833 os_ << "// Copyright (c) 2008-2011 Hartmut Kaiser\n"; 834 os_ << "//\n"; 835 os_ << "// Distributed under the Boost Software License, " 836 "Version 1.0. (See accompanying\n"; 837 os_ << "// file licence_1_0.txt or copy at " 838 "http://www.boost.org/LICENSE_1_0.txt)\n\n"; 839 os_ << "// Auto-generated by boost::lexer, do not edit\n\n"; 840 841 std::string guard(name_suffix); 842 guard += name_suffix[0] ? "_" : ""; 843 guard += __DATE__ "_" __TIME__; 844 std::string::size_type p = guard.find_first_of(": "); 845 while (std::string::npos != p) 846 { 847 guard.replace(p, 1, "_"); 848 p = guard.find_first_of(": ", p); 849 } 850 boost::to_upper(guard); 851 852 os_ << "#if !defined(BOOST_SPIRIT_LEXER_NEXT_TOKEN_" << guard << ")\n"; 853 os_ << "#define BOOST_SPIRIT_LEXER_NEXT_TOKEN_" << guard << "\n\n"; 854 855 os_ << "#include <boost/detail/iterator.hpp>\n"; 856 os_ << "#include <boost/spirit/home/support/detail/lexer/char_traits.hpp>\n\n"; 857 858 generate_delimiter(os_); 859 os_ << "// the generated table of state names and the tokenizer have to be\n" 860 "// defined in the boost::spirit::lex::lexertl::static_ namespace\n"; 861 os_ << "namespace boost { namespace spirit { namespace lex { " 862 "namespace lexertl { namespace static_ {\n\n"; 863 864 // generate the lexer state information variables 865 if (!generate_cpp_state_info(rules_, os_, name_suffix)) 866 return false; 867 868 generate_delimiter(os_); 869 os_ << "// this function returns the next matched token\n"; 870 os_ << "template<typename Iterator>\n"; 871 os_ << "std::size_t next_token" << (name_suffix[0] ? "_" : "") 872 << name_suffix << " ("; 873 874 if (dfas_ > 1) 875 { 876 os_ << "std::size_t& start_state_, "; 877 } 878 else 879 { 880 os_ << "std::size_t& /*start_state_*/, "; 881 } 882 if (sm_.data()._seen_BOL_assertion) 883 { 884 os_ << "bool& bol_, "; 885 } 886 else 887 { 888 os_ << "bool& /*bol_*/, "; 889 } 890 os_ << "\n "; 891 892 os_ << "Iterator &start_token_, Iterator const& end_, "; 893 os_ << "std::size_t& unique_id_)\n"; 894 os_ << "{\n"; 895 if (!generate_function_body(os_, sm_)) 896 return false; 897 os_ << "}\n\n"; 898 899 if (!generate_cpp_state_table(os_, name_suffix 900 , sm_.data()._seen_BOL_assertion, sm_.data()._seen_EOL_assertion)) 901 { 902 return false; 903 } 904 905 os_ << "}}}}} // namespace boost::spirit::lex::lexertl::static_\n\n"; 906 907 os_ << "#endif\n"; 908 909 return os_.good(); 910 } 911 912 } // namespace detail 913 914 /////////////////////////////////////////////////////////////////////////// 915 template <typename Lexer, typename F> 916 inline bool 917 generate_static(Lexer const& lexer, std::ostream& os 918 , char const* name_suffix, F f) 919 { 920 if (!lexer.init_dfa(true)) // always minimize DFA for static lexers 921 return false; 922 return detail::generate_cpp(lexer.state_machine_, lexer.rules_, os 923 , name_suffix, f); 924 } 925 926 /////////////////////////////////////////////////////////////////////////// 927 // deprecated function, will be removed in the future (this has been 928 // replaced by the function generate_static_dfa - see below). 929 template <typename Lexer> 930 inline bool 931 generate_static(Lexer const& lexer, std::ostream& os 932 , char const* name_suffix = "") 933 { 934 return generate_static(lexer, os, name_suffix 935 , &detail::generate_function_body_dfa<typename Lexer::char_type>); 936 } 937 938 /////////////////////////////////////////////////////////////////////////// 939 template <typename Lexer> 940 inline bool 941 generate_static_dfa(Lexer const& lexer, std::ostream& os 942 , char const* name_suffix = "") 943 { 944 return generate_static(lexer, os, name_suffix 945 , &detail::generate_function_body_dfa<typename Lexer::char_type>); 946 } 947 948 /////////////////////////////////////////////////////////////////////////// 949 template <typename Lexer> 950 inline bool 951 generate_static_switch(Lexer const& lexer, std::ostream& os 952 , char const* name_suffix = "") 953 { 954 return generate_static(lexer, os, name_suffix 955 , &detail::generate_function_body_switch<typename Lexer::char_type>); 956 } 957 958/////////////////////////////////////////////////////////////////////////////// 959}}}} 960 961#endif