/src/libyaml/scanner.c
C | 2455 lines | 1043 code | 552 blank | 860 comment | 247 complexity | c13f0124b3e2df97f153328bd89f1a6e MD5 | raw file
Large files files are truncated, but you can click here to view the full file
1 2/* 3 * Introduction 4 * ************ 5 * 6 * The following notes assume that you are familiar with the YAML specification 7 * (http://yaml.org/spec/cvs/current.html). We mostly follow it, although in 8 * some cases we are less restrictive that it requires. 9 * 10 * The process of transforming a YAML stream into a sequence of events is 11 * divided on two steps: Scanning and Parsing. 12 * 13 * The Scanner transforms the input stream into a sequence of tokens, while the 14 * parser transform the sequence of tokens produced by the Scanner into a 15 * sequence of parsing events. 16 * 17 * The Scanner is rather clever and complicated. The Parser, on the contrary, 18 * is a straightforward implementation of a recursive-descendant parser (or, 19 * LL(1) parser, as it is usually called). 20 * 21 * Actually there are two issues of Scanning that might be called "clever", the 22 * rest is quite straightforward. The issues are "block collection start" and 23 * "simple keys". Both issues are explained below in details. 24 * 25 * Here the Scanning step is explained and implemented. We start with the list 26 * of all the tokens produced by the Scanner together with short descriptions. 27 * 28 * Now, tokens: 29 * 30 * STREAM-START(encoding) # The stream start. 31 * STREAM-END # The stream end. 32 * VERSION-DIRECTIVE(major,minor) # The '%YAML' directive. 33 * TAG-DIRECTIVE(handle,prefix) # The '%TAG' directive. 34 * DOCUMENT-START # '---' 35 * DOCUMENT-END # '...' 36 * BLOCK-SEQUENCE-START # Indentation increase denoting a block 37 * BLOCK-MAPPING-START # sequence or a block mapping. 38 * BLOCK-END # Indentation decrease. 39 * FLOW-SEQUENCE-START # '[' 40 * FLOW-SEQUENCE-END # ']' 41 * BLOCK-SEQUENCE-START # '{' 42 * BLOCK-SEQUENCE-END # '}' 43 * BLOCK-ENTRY # '-' 44 * FLOW-ENTRY # ',' 45 * KEY # '?' or nothing (simple keys). 46 * VALUE # ':' 47 * ALIAS(anchor) # '*anchor' 48 * ANCHOR(anchor) # '&anchor' 49 * TAG(handle,suffix) # '!handle!suffix' 50 * SCALAR(value,style) # A scalar. 51 * 52 * The following two tokens are "virtual" tokens denoting the beginning and the 53 * end of the stream: 54 * 55 * STREAM-START(encoding) 56 * STREAM-END 57 * 58 * We pass the information about the input stream encoding with the 59 * STREAM-START token. 60 * 61 * The next two tokens are responsible for tags: 62 * 63 * VERSION-DIRECTIVE(major,minor) 64 * TAG-DIRECTIVE(handle,prefix) 65 * 66 * Example: 67 * 68 * %YAML 1.1 69 * %TAG ! !foo 70 * %TAG !yaml! tag:yaml.org,2002: 71 * --- 72 * 73 * The correspoding sequence of tokens: 74 * 75 * STREAM-START(utf-8) 76 * VERSION-DIRECTIVE(1,1) 77 * TAG-DIRECTIVE("!","!foo") 78 * TAG-DIRECTIVE("!yaml","tag:yaml.org,2002:") 79 * DOCUMENT-START 80 * STREAM-END 81 * 82 * Note that the VERSION-DIRECTIVE and TAG-DIRECTIVE tokens occupy a whole 83 * line. 84 * 85 * The document start and end indicators are represented by: 86 * 87 * DOCUMENT-START 88 * DOCUMENT-END 89 * 90 * Note that if a YAML stream contains an implicit document (without '---' 91 * and '...' indicators), no DOCUMENT-START and DOCUMENT-END tokens will be 92 * produced. 93 * 94 * In the following examples, we present whole documents together with the 95 * produced tokens. 96 * 97 * 1. An implicit document: 98 * 99 * 'a scalar' 100 * 101 * Tokens: 102 * 103 * STREAM-START(utf-8) 104 * SCALAR("a scalar",single-quoted) 105 * STREAM-END 106 * 107 * 2. An explicit document: 108 * 109 * --- 110 * 'a scalar' 111 * ... 112 * 113 * Tokens: 114 * 115 * STREAM-START(utf-8) 116 * DOCUMENT-START 117 * SCALAR("a scalar",single-quoted) 118 * DOCUMENT-END 119 * STREAM-END 120 * 121 * 3. Several documents in a stream: 122 * 123 * 'a scalar' 124 * --- 125 * 'another scalar' 126 * --- 127 * 'yet another scalar' 128 * 129 * Tokens: 130 * 131 * STREAM-START(utf-8) 132 * SCALAR("a scalar",single-quoted) 133 * DOCUMENT-START 134 * SCALAR("another scalar",single-quoted) 135 * DOCUMENT-START 136 * SCALAR("yet another scalar",single-quoted) 137 * STREAM-END 138 * 139 * We have already introduced the SCALAR token above. The following tokens are 140 * used to describe aliases, anchors, tag, and scalars: 141 * 142 * ALIAS(anchor) 143 * ANCHOR(anchor) 144 * TAG(handle,suffix) 145 * SCALAR(value,style) 146 * 147 * The following series of examples illustrate the usage of these tokens: 148 * 149 * 1. A recursive sequence: 150 * 151 * &A [ *A ] 152 * 153 * Tokens: 154 * 155 * STREAM-START(utf-8) 156 * ANCHOR("A") 157 * FLOW-SEQUENCE-START 158 * ALIAS("A") 159 * FLOW-SEQUENCE-END 160 * STREAM-END 161 * 162 * 2. A tagged scalar: 163 * 164 * !!float "3.14" # A good approximation. 165 * 166 * Tokens: 167 * 168 * STREAM-START(utf-8) 169 * TAG("!!","float") 170 * SCALAR("3.14",double-quoted) 171 * STREAM-END 172 * 173 * 3. Various scalar styles: 174 * 175 * --- # Implicit empty plain scalars do not produce tokens. 176 * --- a plain scalar 177 * --- 'a single-quoted scalar' 178 * --- "a double-quoted scalar" 179 * --- |- 180 * a literal scalar 181 * --- >- 182 * a folded 183 * scalar 184 * 185 * Tokens: 186 * 187 * STREAM-START(utf-8) 188 * DOCUMENT-START 189 * DOCUMENT-START 190 * SCALAR("a plain scalar",plain) 191 * DOCUMENT-START 192 * SCALAR("a single-quoted scalar",single-quoted) 193 * DOCUMENT-START 194 * SCALAR("a double-quoted scalar",double-quoted) 195 * DOCUMENT-START 196 * SCALAR("a literal scalar",literal) 197 * DOCUMENT-START 198 * SCALAR("a folded scalar",folded) 199 * STREAM-END 200 * 201 * Now it's time to review collection-related tokens. We will start with 202 * flow collections: 203 * 204 * FLOW-SEQUENCE-START 205 * FLOW-SEQUENCE-END 206 * FLOW-MAPPING-START 207 * FLOW-MAPPING-END 208 * FLOW-ENTRY 209 * KEY 210 * VALUE 211 * 212 * The tokens FLOW-SEQUENCE-START, FLOW-SEQUENCE-END, FLOW-MAPPING-START, and 213 * FLOW-MAPPING-END represent the indicators '[', ']', '{', and '}' 214 * correspondingly. FLOW-ENTRY represent the ',' indicator. Finally the 215 * indicators '?' and ':', which are used for denoting mapping keys and values, 216 * are represented by the KEY and VALUE tokens. 217 * 218 * The following examples show flow collections: 219 * 220 * 1. A flow sequence: 221 * 222 * [item 1, item 2, item 3] 223 * 224 * Tokens: 225 * 226 * STREAM-START(utf-8) 227 * FLOW-SEQUENCE-START 228 * SCALAR("item 1",plain) 229 * FLOW-ENTRY 230 * SCALAR("item 2",plain) 231 * FLOW-ENTRY 232 * SCALAR("item 3",plain) 233 * FLOW-SEQUENCE-END 234 * STREAM-END 235 * 236 * 2. A flow mapping: 237 * 238 * { 239 * a simple key: a value, # Note that the KEY token is produced. 240 * ? a complex key: another value, 241 * } 242 * 243 * Tokens: 244 * 245 * STREAM-START(utf-8) 246 * FLOW-MAPPING-START 247 * KEY 248 * SCALAR("a simple key",plain) 249 * VALUE 250 * SCALAR("a value",plain) 251 * FLOW-ENTRY 252 * KEY 253 * SCALAR("a complex key",plain) 254 * VALUE 255 * SCALAR("another value",plain) 256 * FLOW-ENTRY 257 * FLOW-MAPPING-END 258 * STREAM-END 259 * 260 * A simple key is a key which is not denoted by the '?' indicator. Note that 261 * the Scanner still produce the KEY token whenever it encounters a simple key. 262 * 263 * For scanning block collections, the following tokens are used (note that we 264 * repeat KEY and VALUE here): 265 * 266 * BLOCK-SEQUENCE-START 267 * BLOCK-MAPPING-START 268 * BLOCK-END 269 * BLOCK-ENTRY 270 * KEY 271 * VALUE 272 * 273 * The tokens BLOCK-SEQUENCE-START and BLOCK-MAPPING-START denote indentation 274 * increase that precedes a block collection (cf. the INDENT token in Python). 275 * The token BLOCK-END denote indentation decrease that ends a block collection 276 * (cf. the DEDENT token in Python). However YAML has some syntax pecularities 277 * that makes detections of these tokens more complex. 278 * 279 * The tokens BLOCK-ENTRY, KEY, and VALUE are used to represent the indicators 280 * '-', '?', and ':' correspondingly. 281 * 282 * The following examples show how the tokens BLOCK-SEQUENCE-START, 283 * BLOCK-MAPPING-START, and BLOCK-END are emitted by the Scanner: 284 * 285 * 1. Block sequences: 286 * 287 * - item 1 288 * - item 2 289 * - 290 * - item 3.1 291 * - item 3.2 292 * - 293 * key 1: value 1 294 * key 2: value 2 295 * 296 * Tokens: 297 * 298 * STREAM-START(utf-8) 299 * BLOCK-SEQUENCE-START 300 * BLOCK-ENTRY 301 * SCALAR("item 1",plain) 302 * BLOCK-ENTRY 303 * SCALAR("item 2",plain) 304 * BLOCK-ENTRY 305 * BLOCK-SEQUENCE-START 306 * BLOCK-ENTRY 307 * SCALAR("item 3.1",plain) 308 * BLOCK-ENTRY 309 * SCALAR("item 3.2",plain) 310 * BLOCK-END 311 * BLOCK-ENTRY 312 * BLOCK-MAPPING-START 313 * KEY 314 * SCALAR("key 1",plain) 315 * VALUE 316 * SCALAR("value 1",plain) 317 * KEY 318 * SCALAR("key 2",plain) 319 * VALUE 320 * SCALAR("value 2",plain) 321 * BLOCK-END 322 * BLOCK-END 323 * STREAM-END 324 * 325 * 2. Block mappings: 326 * 327 * a simple key: a value # The KEY token is produced here. 328 * ? a complex key 329 * : another value 330 * a mapping: 331 * key 1: value 1 332 * key 2: value 2 333 * a sequence: 334 * - item 1 335 * - item 2 336 * 337 * Tokens: 338 * 339 * STREAM-START(utf-8) 340 * BLOCK-MAPPING-START 341 * KEY 342 * SCALAR("a simple key",plain) 343 * VALUE 344 * SCALAR("a value",plain) 345 * KEY 346 * SCALAR("a complex key",plain) 347 * VALUE 348 * SCALAR("another value",plain) 349 * KEY 350 * SCALAR("a mapping",plain) 351 * BLOCK-MAPPING-START 352 * KEY 353 * SCALAR("key 1",plain) 354 * VALUE 355 * SCALAR("value 1",plain) 356 * KEY 357 * SCALAR("key 2",plain) 358 * VALUE 359 * SCALAR("value 2",plain) 360 * BLOCK-END 361 * KEY 362 * SCALAR("a sequence",plain) 363 * VALUE 364 * BLOCK-SEQUENCE-START 365 * BLOCK-ENTRY 366 * SCALAR("item 1",plain) 367 * BLOCK-ENTRY 368 * SCALAR("item 2",plain) 369 * BLOCK-END 370 * BLOCK-END 371 * STREAM-END 372 * 373 * YAML does not always require to start a new block collection from a new 374 * line. If the current line contains only '-', '?', and ':' indicators, a new 375 * block collection may start at the current line. The following examples 376 * illustrate this case: 377 * 378 * 1. Collections in a sequence: 379 * 380 * - - item 1 381 * - item 2 382 * - key 1: value 1 383 * key 2: value 2 384 * - ? complex key 385 * : complex value 386 * 387 * Tokens: 388 * 389 * STREAM-START(utf-8) 390 * BLOCK-SEQUENCE-START 391 * BLOCK-ENTRY 392 * BLOCK-SEQUENCE-START 393 * BLOCK-ENTRY 394 * SCALAR("item 1",plain) 395 * BLOCK-ENTRY 396 * SCALAR("item 2",plain) 397 * BLOCK-END 398 * BLOCK-ENTRY 399 * BLOCK-MAPPING-START 400 * KEY 401 * SCALAR("key 1",plain) 402 * VALUE 403 * SCALAR("value 1",plain) 404 * KEY 405 * SCALAR("key 2",plain) 406 * VALUE 407 * SCALAR("value 2",plain) 408 * BLOCK-END 409 * BLOCK-ENTRY 410 * BLOCK-MAPPING-START 411 * KEY 412 * SCALAR("complex key") 413 * VALUE 414 * SCALAR("complex value") 415 * BLOCK-END 416 * BLOCK-END 417 * STREAM-END 418 * 419 * 2. Collections in a mapping: 420 * 421 * ? a sequence 422 * : - item 1 423 * - item 2 424 * ? a mapping 425 * : key 1: value 1 426 * key 2: value 2 427 * 428 * Tokens: 429 * 430 * STREAM-START(utf-8) 431 * BLOCK-MAPPING-START 432 * KEY 433 * SCALAR("a sequence",plain) 434 * VALUE 435 * BLOCK-SEQUENCE-START 436 * BLOCK-ENTRY 437 * SCALAR("item 1",plain) 438 * BLOCK-ENTRY 439 * SCALAR("item 2",plain) 440 * BLOCK-END 441 * KEY 442 * SCALAR("a mapping",plain) 443 * VALUE 444 * BLOCK-MAPPING-START 445 * KEY 446 * SCALAR("key 1",plain) 447 * VALUE 448 * SCALAR("value 1",plain) 449 * KEY 450 * SCALAR("key 2",plain) 451 * VALUE 452 * SCALAR("value 2",plain) 453 * BLOCK-END 454 * BLOCK-END 455 * STREAM-END 456 * 457 * YAML also permits non-indented sequences if they are included into a block 458 * mapping. In this case, the token BLOCK-SEQUENCE-START is not produced: 459 * 460 * key: 461 * - item 1 # BLOCK-SEQUENCE-START is NOT produced here. 462 * - item 2 463 * 464 * Tokens: 465 * 466 * STREAM-START(utf-8) 467 * BLOCK-MAPPING-START 468 * KEY 469 * SCALAR("key",plain) 470 * VALUE 471 * BLOCK-ENTRY 472 * SCALAR("item 1",plain) 473 * BLOCK-ENTRY 474 * SCALAR("item 2",plain) 475 * BLOCK-END 476 */ 477 478#include "yaml_private.h" 479 480/* 481 * Ensure that the buffer contains the required number of characters. 482 * Return 1 on success, 0 on failure (reader error or memory error). 483 */ 484 485#define CACHE(parser,length) \ 486 (parser->unread >= (length) \ 487 ? 1 \ 488 : yaml_parser_update_buffer(parser, (length))) 489 490/* 491 * Advance the buffer pointer. 492 */ 493 494#define SKIP(parser) \ 495 (parser->mark.index ++, \ 496 parser->mark.column ++, \ 497 parser->unread --, \ 498 parser->buffer.pointer ++) 499 500#define SKIPN(parser,n) \ 501 (parser->mark.index ++, \ 502 parser->mark.column ++, \ 503 parser->unread --, \ 504 parser->buffer.pointer += (n)) 505#define SKIP_LINE(parser) \ 506 (IS_CRLF(parser->buffer) ? \ 507 (parser->mark.index += 2, \ 508 parser->mark.column = 0, \ 509 parser->mark.line ++, \ 510 parser->unread -= 2, \ 511 parser->buffer.pointer += 2) : \ 512 IS_BREAK(parser->buffer) ? \ 513 (parser->mark.index ++, \ 514 parser->mark.column = 0, \ 515 parser->mark.line ++, \ 516 parser->unread --, \ 517 parser->buffer.pointer += WIDTH(parser->buffer)) : 0) 518 519/* 520 * Copy a character to a string buffer and advance pointers. 521 */ 522 523#define READ(parser,string) \ 524 (STRING_EXTEND(parser,string) ? \ 525 (COPY(string,parser->buffer), \ 526 parser->mark.index ++, \ 527 parser->mark.column ++, \ 528 parser->unread --, \ 529 1) : 0) 530 531#define READN(parser,string,n) \ 532 (STRING_EXTEND(parser,string) ? \ 533 (COPYN(string,parser->buffer,n), \ 534 parser->mark.index ++, \ 535 parser->mark.column ++, \ 536 parser->unread --, \ 537 1) : 0) 538 539/* 540 * Copy a line break character to a string buffer and advance pointers. 541 */ 542 543#define READ_LINE(parser,string) \ 544 (STRING_EXTEND(parser,string) ? \ 545 (((CHECK_AT(parser->buffer,'\r',0) \ 546 && CHECK_AT(parser->buffer,'\n',1)) ? /* CR LF -> LF */ \ 547 (*((string).pointer++) = (yaml_char_t) '\n', \ 548 parser->buffer.pointer += 2, \ 549 parser->mark.index += 2, \ 550 parser->mark.column = 0, \ 551 parser->mark.line ++, \ 552 parser->unread -= 2) : \ 553 (CHECK_AT(parser->buffer,'\r',0) \ 554 || CHECK_AT(parser->buffer,'\n',0)) ? /* CR|LF -> LF */ \ 555 (*((string).pointer++) = (yaml_char_t) '\n', \ 556 parser->buffer.pointer ++, \ 557 parser->mark.index ++, \ 558 parser->mark.column = 0, \ 559 parser->mark.line ++, \ 560 parser->unread --) : \ 561 (CHECK_AT(parser->buffer,'\xC2',0) \ 562 && CHECK_AT(parser->buffer,'\x85',1)) ? /* NEL -> LF */ \ 563 (*((string).pointer++) = (yaml_char_t) '\n', \ 564 parser->buffer.pointer += 2, \ 565 parser->mark.index ++, \ 566 parser->mark.column = 0, \ 567 parser->mark.line ++, \ 568 parser->unread --) : \ 569 (CHECK_AT(parser->buffer,'\xE2',0) && \ 570 CHECK_AT(parser->buffer,'\x80',1) && \ 571 (CHECK_AT(parser->buffer,'\xA8',2) || \ 572 CHECK_AT(parser->buffer,'\xA9',2))) ? /* LS|PS -> LS|PS */ \ 573 (*((string).pointer++) = *(parser->buffer.pointer++), \ 574 *((string).pointer++) = *(parser->buffer.pointer++), \ 575 *((string).pointer++) = *(parser->buffer.pointer++), \ 576 parser->mark.index ++, \ 577 parser->mark.column = 0, \ 578 parser->mark.line ++, \ 579 parser->unread --) : 0), \ 580 1) : 0) 581 582/* 583 * Public API declarations. 584 */ 585 586YAML_DECLARE(int) 587yaml_parser_scan(yaml_parser_t *parser, yaml_token_t *token); 588 589/* 590 * Error handling. 591 */ 592 593static int 594yaml_parser_set_scanner_error(yaml_parser_t *parser, const char *context, 595 yaml_mark_t context_mark, const char *problem); 596 597/* 598 * High-level token API. 599 */ 600 601YAML_DECLARE(int) 602yaml_parser_fetch_more_tokens(yaml_parser_t *parser); 603 604static int 605yaml_parser_fetch_next_token(yaml_parser_t *parser); 606 607/* 608 * Potential simple keys. 609 */ 610 611static int 612yaml_parser_stale_simple_keys(yaml_parser_t *parser); 613 614static int 615yaml_parser_save_simple_key(yaml_parser_t *parser); 616 617static int 618yaml_parser_remove_simple_key(yaml_parser_t *parser); 619 620static int 621yaml_parser_increase_flow_level(yaml_parser_t *parser); 622 623static int 624yaml_parser_decrease_flow_level(yaml_parser_t *parser); 625 626/* 627 * Indentation treatment. 628 */ 629 630static int 631yaml_parser_roll_indent(yaml_parser_t *parser, int column, 632 int number, yaml_token_type_t type, yaml_mark_t mark); 633 634static int 635yaml_parser_unroll_indent(yaml_parser_t *parser, int column); 636 637/* 638 * Token fetchers. 639 */ 640 641static int 642yaml_parser_fetch_stream_start(yaml_parser_t *parser); 643 644static int 645yaml_parser_fetch_stream_end(yaml_parser_t *parser); 646 647static int 648yaml_parser_fetch_directive(yaml_parser_t *parser); 649 650static int 651yaml_parser_fetch_document_indicator(yaml_parser_t *parser, 652 yaml_token_type_t type); 653 654static int 655yaml_parser_fetch_flow_collection_start(yaml_parser_t *parser, 656 yaml_token_type_t type); 657 658static int 659yaml_parser_fetch_flow_collection_end(yaml_parser_t *parser, 660 yaml_token_type_t type); 661 662static int 663yaml_parser_fetch_flow_entry(yaml_parser_t *parser); 664 665static int 666yaml_parser_fetch_block_entry(yaml_parser_t *parser); 667 668static int 669yaml_parser_fetch_key(yaml_parser_t *parser); 670 671static int 672yaml_parser_fetch_value(yaml_parser_t *parser); 673 674static int 675yaml_parser_fetch_anchor(yaml_parser_t *parser, yaml_token_type_t type); 676 677static int 678yaml_parser_fetch_tag(yaml_parser_t *parser); 679 680static int 681yaml_parser_fetch_block_scalar(yaml_parser_t *parser, int literal); 682 683static int 684yaml_parser_fetch_flow_scalar(yaml_parser_t *parser, int single); 685 686static int 687yaml_parser_fetch_plain_scalar(yaml_parser_t *parser); 688 689/* 690 * Token scanners. 691 */ 692 693static int 694yaml_parser_scan_to_next_token(yaml_parser_t *parser); 695 696static int 697yaml_parser_scan_directive(yaml_parser_t *parser, yaml_token_t *token); 698 699static int 700yaml_parser_scan_directive_name(yaml_parser_t *parser, 701 yaml_mark_t start_mark, yaml_char_t **name); 702 703static int 704yaml_parser_scan_version_directive_value(yaml_parser_t *parser, 705 yaml_mark_t start_mark, int *major, int *minor); 706 707static int 708yaml_parser_scan_version_directive_number(yaml_parser_t *parser, 709 yaml_mark_t start_mark, int *number); 710 711static int 712yaml_parser_scan_tag_directive_value(yaml_parser_t *parser, 713 yaml_mark_t mark, yaml_char_t **handle, yaml_char_t **prefix); 714 715static int 716yaml_parser_scan_anchor(yaml_parser_t *parser, yaml_token_t *token, 717 yaml_token_type_t type); 718 719static int 720yaml_parser_scan_tag(yaml_parser_t *parser, yaml_token_t *token); 721 722static int 723yaml_parser_scan_tag_handle(yaml_parser_t *parser, int directive, 724 yaml_mark_t start_mark, yaml_char_t **handle); 725 726static int 727yaml_parser_scan_tag_uri(yaml_parser_t *parser, int directive, 728 yaml_char_t *head, yaml_mark_t start_mark, yaml_char_t **uri); 729 730static int 731yaml_parser_scan_uri_escapes(yaml_parser_t *parser, int directive, 732 yaml_mark_t start_mark, yaml_string_t *string); 733 734static int 735yaml_parser_scan_block_scalar(yaml_parser_t *parser, yaml_token_t *token, 736 int literal); 737 738static int 739yaml_parser_scan_block_scalar_breaks(yaml_parser_t *parser, 740 int *indent, yaml_string_t *breaks, 741 yaml_mark_t start_mark, yaml_mark_t *end_mark); 742 743static int 744yaml_parser_scan_flow_scalar(yaml_parser_t *parser, yaml_token_t *token, 745 int single); 746 747static int 748yaml_parser_scan_plain_scalar(yaml_parser_t *parser, yaml_token_t *token); 749 750/* 751 * Get the next token. 752 */ 753 754YAML_DECLARE(int) 755yaml_parser_scan(yaml_parser_t *parser, yaml_token_t *token) 756{ 757 assert(parser); /* Non-NULL parser object is expected. */ 758 assert(token); /* Non-NULL token object is expected. */ 759 760 /* Erase the token object. */ 761 762 memset(token, 0, sizeof(yaml_token_t)); 763 764 /* No tokens after STREAM-END or error. */ 765 766 if (parser->stream_end_produced || parser->error) { 767 return 1; 768 } 769 770 /* Ensure that the tokens queue contains enough tokens. */ 771 772 if (!parser->token_available) { 773 if (!yaml_parser_fetch_more_tokens(parser)) 774 return 0; 775 } 776 777 /* Fetch the next token from the queue. */ 778 779 *token = DEQUEUE(parser, parser->tokens); 780 parser->token_available = 0; 781 parser->tokens_parsed ++; 782 783 if (token->type == YAML_STREAM_END_TOKEN) { 784 parser->stream_end_produced = 1; 785 } 786 787 return 1; 788} 789 790/* 791 * Set the scanner error and return 0. 792 */ 793 794static int 795yaml_parser_set_scanner_error(yaml_parser_t *parser, const char *context, 796 yaml_mark_t context_mark, const char *problem) 797{ 798 parser->error = YAML_SCANNER_ERROR; 799 parser->context = context; 800 parser->context_mark = context_mark; 801 parser->problem = problem; 802 parser->problem_mark = parser->mark; 803 804 return 0; 805} 806 807/* 808 * Ensure that the tokens queue contains at least one token which can be 809 * returned to the Parser. 810 */ 811 812YAML_DECLARE(int) 813yaml_parser_fetch_more_tokens(yaml_parser_t *parser) 814{ 815 int need_more_tokens; 816 817 /* While we need more tokens to fetch, do it. */ 818 819 while (1) 820 { 821 /* 822 * Check if we really need to fetch more tokens. 823 */ 824 825 need_more_tokens = 0; 826 827 if (parser->tokens.head == parser->tokens.tail) 828 { 829 /* Queue is empty. */ 830 831 need_more_tokens = 1; 832 } 833 else 834 { 835 yaml_simple_key_t *simple_key; 836 837 /* Check if any potential simple key may occupy the head position. */ 838 839 if (!yaml_parser_stale_simple_keys(parser)) 840 return 0; 841 842 for (simple_key = parser->simple_keys.start; 843 simple_key != parser->simple_keys.top; simple_key++) { 844 if (simple_key->possible 845 && simple_key->token_number == parser->tokens_parsed) { 846 need_more_tokens = 1; 847 break; 848 } 849 } 850 } 851 852 /* We are finished. */ 853 854 if (!need_more_tokens) 855 break; 856 857 /* Fetch the next token. */ 858 859 if (!yaml_parser_fetch_next_token(parser)) 860 return 0; 861 } 862 863 parser->token_available = 1; 864 865 return 1; 866} 867 868/* 869 * The dispatcher for token fetchers. 870 */ 871 872static int 873yaml_parser_fetch_next_token(yaml_parser_t *parser) 874{ 875 /* Ensure that the buffer is initialized. */ 876 877 if (!CACHE(parser, 1)) 878 return 0; 879 880 /* Check if we just started scanning. Fetch STREAM-START then. */ 881 882 if (!parser->stream_start_produced) 883 return yaml_parser_fetch_stream_start(parser); 884 885 /* Eat whitespaces and comments until we reach the next token. */ 886 887 if (!yaml_parser_scan_to_next_token(parser)) 888 return 0; 889 890 /* Remove obsolete potential simple keys. */ 891 892 if (!yaml_parser_stale_simple_keys(parser)) 893 return 0; 894 895 /* Check the indentation level against the current column. */ 896 897 if (!yaml_parser_unroll_indent(parser, parser->mark.column)) 898 return 0; 899 900 /* 901 * Ensure that the buffer contains at least 4 characters. 4 is the length 902 * of the longest indicators ('--- ' and '... '). 903 */ 904 905 if (!CACHE(parser, 4)) 906 return 0; 907 908 /* Is it the end of the stream? */ 909 910 if (IS_Z(parser->buffer)) 911 return yaml_parser_fetch_stream_end(parser); 912 913 /* Is it a directive? */ 914 915 if (parser->mark.column == 0 && CHECK(parser->buffer, '%')) 916 return yaml_parser_fetch_directive(parser); 917 918 /* Is it the document start indicator? */ 919 920 if (parser->mark.column == 0 921 && CHECK_AT(parser->buffer, '-', 0) 922 && CHECK_AT(parser->buffer, '-', 1) 923 && CHECK_AT(parser->buffer, '-', 2) 924 && IS_BLANKZ_AT(parser->buffer, 3)) 925 return yaml_parser_fetch_document_indicator(parser, 926 YAML_DOCUMENT_START_TOKEN); 927 928 /* Is it the document end indicator? */ 929 930 if (parser->mark.column == 0 931 && CHECK_AT(parser->buffer, '.', 0) 932 && CHECK_AT(parser->buffer, '.', 1) 933 && CHECK_AT(parser->buffer, '.', 2) 934 && IS_BLANKZ_AT(parser->buffer, 3)) 935 return yaml_parser_fetch_document_indicator(parser, 936 YAML_DOCUMENT_END_TOKEN); 937 938 /* Is it the flow sequence start indicator? */ 939 940 if (CHECK(parser->buffer, '[')) 941 return yaml_parser_fetch_flow_collection_start(parser, 942 YAML_FLOW_SEQUENCE_START_TOKEN); 943 944 /* Is it the flow mapping start indicator? */ 945 946 if (CHECK(parser->buffer, '{')) 947 return yaml_parser_fetch_flow_collection_start(parser, 948 YAML_FLOW_MAPPING_START_TOKEN); 949 950 /* Is it the flow sequence end indicator? */ 951 952 if (CHECK(parser->buffer, ']')) 953 return yaml_parser_fetch_flow_collection_end(parser, 954 YAML_FLOW_SEQUENCE_END_TOKEN); 955 956 /* Is it the flow mapping end indicator? */ 957 958 if (CHECK(parser->buffer, '}')) 959 return yaml_parser_fetch_flow_collection_end(parser, 960 YAML_FLOW_MAPPING_END_TOKEN); 961 962 /* Is it the flow entry indicator? */ 963 964 if (CHECK(parser->buffer, ',')) 965 return yaml_parser_fetch_flow_entry(parser); 966 967 /* Is it the block entry indicator? */ 968 969 if (CHECK(parser->buffer, '-') && IS_BLANKZ_AT(parser->buffer, 1)) 970 return yaml_parser_fetch_block_entry(parser); 971 972 /* Is it the key indicator? */ 973 974 if (CHECK(parser->buffer, '?') 975 && (parser->flow_level || IS_BLANKZ_AT(parser->buffer, 1))) 976 return yaml_parser_fetch_key(parser); 977 978 /* Is it the value indicator? */ 979 980 if (CHECK(parser->buffer, ':') 981 && (parser->flow_level || IS_BLANKZ_AT(parser->buffer, 1))) 982 return yaml_parser_fetch_value(parser); 983 984 /* Is it an alias? */ 985 986 if (CHECK(parser->buffer, '*')) 987 return yaml_parser_fetch_anchor(parser, YAML_ALIAS_TOKEN); 988 989 /* Is it an anchor? */ 990 991 if (CHECK(parser->buffer, '&')) 992 return yaml_parser_fetch_anchor(parser, YAML_ANCHOR_TOKEN); 993 994 /* Is it a tag? */ 995 996 if (CHECK(parser->buffer, '!')) 997 return yaml_parser_fetch_tag(parser); 998 999 /* Is it a literal scalar? */ 1000 1001 if (CHECK(parser->buffer, '|') && !parser->flow_level) 1002 return yaml_parser_fetch_block_scalar(parser, 1); 1003 1004 /* Is it a folded scalar? */ 1005 1006 if (CHECK(parser->buffer, '>') && !parser->flow_level) 1007 return yaml_parser_fetch_block_scalar(parser, 0); 1008 1009 /* Is it a single-quoted scalar? */ 1010 1011 if (CHECK(parser->buffer, '\'')) 1012 return yaml_parser_fetch_flow_scalar(parser, 1); 1013 1014 /* Is it a double-quoted scalar? */ 1015 1016 if (CHECK(parser->buffer, '"')) 1017 return yaml_parser_fetch_flow_scalar(parser, 0); 1018 1019 /* 1020 * Is it a plain scalar? 1021 * 1022 * A plain scalar may start with any non-blank characters except 1023 * 1024 * '-', '?', ':', ',', '[', ']', '{', '}', 1025 * '#', '&', '*', '!', '|', '>', '\'', '\"', 1026 * '%', '@', '`'. 1027 * 1028 * In the block context (and, for the '-' indicator, in the flow context 1029 * too), it may also start with the characters 1030 * 1031 * '-', '?', ':' 1032 * 1033 * if it is followed by a non-space character. 1034 * 1035 * The last rule is more restrictive than the specification requires. 1036 */ 1037 1038 if (!(IS_BLANKZ(parser->buffer) || CHECK(parser->buffer, '-') 1039 || CHECK(parser->buffer, '?') || CHECK(parser->buffer, ':') 1040 || CHECK(parser->buffer, ',') || CHECK(parser->buffer, '[') 1041 || CHECK(parser->buffer, ']') || CHECK(parser->buffer, '{') 1042 || CHECK(parser->buffer, '}') || CHECK(parser->buffer, '#') 1043 || CHECK(parser->buffer, '&') || CHECK(parser->buffer, '*') 1044 || CHECK(parser->buffer, '!') || CHECK(parser->buffer, '|') 1045 || CHECK(parser->buffer, '>') || CHECK(parser->buffer, '\'') 1046 || CHECK(parser->buffer, '"') || CHECK(parser->buffer, '%') 1047 || CHECK(parser->buffer, '@') || CHECK(parser->buffer, '`')) || 1048 (CHECK(parser->buffer, '-') && !IS_BLANK_AT(parser->buffer, 1)) || 1049 (!parser->flow_level && 1050 (CHECK(parser->buffer, '?') || CHECK(parser->buffer, ':')) 1051 && !IS_BLANKZ_AT(parser->buffer, 1))) 1052 return yaml_parser_fetch_plain_scalar(parser); 1053 1054 /* 1055 * If we don't determine the token type so far, it is an error. 1056 */ 1057 1058 return yaml_parser_set_scanner_error(parser, 1059 "while scanning for the next token", parser->mark, 1060 "found character that cannot start any token"); 1061} 1062 1063/* 1064 * Check the list of potential simple keys and remove the positions that 1065 * cannot contain simple keys anymore. 1066 */ 1067 1068static int 1069yaml_parser_stale_simple_keys(yaml_parser_t *parser) 1070{ 1071 yaml_simple_key_t *simple_key; 1072 1073 /* Check for a potential simple key for each flow level. */ 1074 1075 for (simple_key = parser->simple_keys.start; 1076 simple_key != parser->simple_keys.top; simple_key ++) 1077 { 1078 /* 1079 * The specification requires that a simple key 1080 * 1081 * - is limited to a single line, 1082 * - is shorter than 1024 characters. 1083 */ 1084 1085 if (simple_key->possible 1086 && (simple_key->mark.line < parser->mark.line 1087 || simple_key->mark.index+1024 < parser->mark.index)) { 1088 1089 /* Check if the potential simple key to be removed is required. */ 1090 1091 if (simple_key->required) { 1092 return yaml_parser_set_scanner_error(parser, 1093 "while scanning a simple key", simple_key->mark, 1094 "could not find expected ':'"); 1095 } 1096 1097 simple_key->possible = 0; 1098 } 1099 } 1100 1101 return 1; 1102} 1103 1104/* 1105 * Check if a simple key may start at the current position and add it if 1106 * needed. 1107 */ 1108 1109static int 1110yaml_parser_save_simple_key(yaml_parser_t *parser) 1111{ 1112 /* 1113 * A simple key is required at the current position if the scanner is in 1114 * the block context and the current column coincides with the indentation 1115 * level. 1116 */ 1117 1118 int required = (!parser->flow_level 1119 && parser->indent == (int)parser->mark.column); 1120 1121 /* 1122 * A simple key is required only when it is the first token in the current 1123 * line. Therefore it is always allowed. But we add a check anyway. 1124 */ 1125 1126 assert(parser->simple_key_allowed || !required); /* Impossible. */ 1127 1128 /* 1129 * If the current position may start a simple key, save it. 1130 */ 1131 1132 if (parser->simple_key_allowed) 1133 { 1134 yaml_simple_key_t simple_key; 1135 simple_key.possible = 1; 1136 simple_key.required = required; 1137 simple_key.token_number = 1138 parser->tokens_parsed + parser->tokens.tail - parser->tokens.head; 1139 simple_key.mark = parser->mark; 1140 1141 if (!yaml_parser_remove_simple_key(parser)) return 0; 1142 1143 *(parser->simple_keys.top-1) = simple_key; 1144 } 1145 1146 return 1; 1147} 1148 1149/* 1150 * Remove a potential simple key at the current flow level. 1151 */ 1152 1153static int 1154yaml_parser_remove_simple_key(yaml_parser_t *parser) 1155{ 1156 yaml_simple_key_t *simple_key = parser->simple_keys.top-1; 1157 1158 if (simple_key->possible) 1159 { 1160 /* If the key is required, it is an error. */ 1161 1162 if (simple_key->required) { 1163 return yaml_parser_set_scanner_error(parser, 1164 "while scanning a simple key", simple_key->mark, 1165 "could not find expected ':'"); 1166 } 1167 } 1168 1169 /* Remove the key from the stack. */ 1170 1171 simple_key->possible = 0; 1172 1173 return 1; 1174} 1175 1176/* 1177 * Increase the flow level and resize the simple key list if needed. 1178 */ 1179 1180static int 1181yaml_parser_increase_flow_level(yaml_parser_t *parser) 1182{ 1183 yaml_simple_key_t empty_simple_key = { 0, 0, 0, { 0, 0, 0 } }; 1184 1185 /* Reset the simple key on the next level. */ 1186 1187 if (!PUSH(parser, parser->simple_keys, empty_simple_key)) 1188 return 0; 1189 1190 /* Increase the flow level. */ 1191 1192 parser->flow_level++; 1193 1194 return 1; 1195} 1196 1197/* 1198 * Decrease the flow level. 1199 */ 1200 1201static int 1202yaml_parser_decrease_flow_level(yaml_parser_t *parser) 1203{ 1204 yaml_simple_key_t dummy_key; /* Used to eliminate a compiler warning. */ 1205 1206 if (parser->flow_level) { 1207 parser->flow_level --; 1208 dummy_key = POP(parser, parser->simple_keys); 1209 } 1210 1211 return 1; 1212} 1213 1214/* 1215 * Push the current indentation level to the stack and set the new level 1216 * the current column is greater than the indentation level. In this case, 1217 * append or insert the specified token into the token queue. 1218 * 1219 */ 1220 1221static int 1222yaml_parser_roll_indent(yaml_parser_t *parser, int column, 1223 int number, yaml_token_type_t type, yaml_mark_t mark) 1224{ 1225 yaml_token_t token; 1226 1227 /* In the flow context, do nothing. */ 1228 1229 if (parser->flow_level) 1230 return 1; 1231 1232 if (parser->indent < column) 1233 { 1234 /* 1235 * Push the current indentation level to the stack and set the new 1236 * indentation level. 1237 */ 1238 1239 if (!PUSH(parser, parser->indents, parser->indent)) 1240 return 0; 1241 1242 parser->indent = column; 1243 1244 /* Create a token and insert it into the queue. */ 1245 1246 TOKEN_INIT(token, type, mark, mark); 1247 1248 if (number == -1) { 1249 if (!ENQUEUE(parser, parser->tokens, token)) 1250 return 0; 1251 } 1252 else { 1253 if (!QUEUE_INSERT(parser, 1254 parser->tokens, number - parser->tokens_parsed, token)) 1255 return 0; 1256 } 1257 } 1258 1259 return 1; 1260} 1261 1262/* 1263 * Pop indentation levels from the indents stack until the current level 1264 * becomes less or equal to the column. For each intendation level, append 1265 * the BLOCK-END token. 1266 */ 1267 1268 1269static int 1270yaml_parser_unroll_indent(yaml_parser_t *parser, int column) 1271{ 1272 yaml_token_t token; 1273 1274 /* In the flow context, do nothing. */ 1275 1276 if (parser->flow_level) 1277 return 1; 1278 1279 /* Loop through the intendation levels in the stack. */ 1280 1281 while (parser->indent > column) 1282 { 1283 /* Create a token and append it to the queue. */ 1284 1285 TOKEN_INIT(token, YAML_BLOCK_END_TOKEN, parser->mark, parser->mark); 1286 1287 if (!ENQUEUE(parser, parser->tokens, token)) 1288 return 0; 1289 1290 /* Pop the indentation level. */ 1291 1292 parser->indent = POP(parser, parser->indents); 1293 } 1294 1295 return 1; 1296} 1297 1298/* 1299 * Initialize the scanner and produce the STREAM-START token. 1300 */ 1301 1302static int 1303yaml_parser_fetch_stream_start(yaml_parser_t *parser) 1304{ 1305 yaml_simple_key_t simple_key = { 0, 0, 0, { 0, 0, 0 } }; 1306 yaml_token_t token; 1307 1308 /* Set the initial indentation. */ 1309 1310 parser->indent = -1; 1311 1312 /* Initialize the simple key stack. */ 1313 1314 if (!PUSH(parser, parser->simple_keys, simple_key)) 1315 return 0; 1316 1317 /* A simple key is allowed at the beginning of the stream. */ 1318 1319 parser->simple_key_allowed = 1; 1320 1321 /* We have started. */ 1322 1323 parser->stream_start_produced = 1; 1324 1325 /* Create the STREAM-START token and append it to the queue. */ 1326 1327 STREAM_START_TOKEN_INIT(token, parser->encoding, 1328 parser->mark, parser->mark); 1329 1330 if (!ENQUEUE(parser, parser->tokens, token)) 1331 return 0; 1332 1333 return 1; 1334} 1335 1336/* 1337 * Produce the STREAM-END token and shut down the scanner. 1338 */ 1339 1340static int 1341yaml_parser_fetch_stream_end(yaml_parser_t *parser) 1342{ 1343 yaml_token_t token; 1344 1345 /* Force new line. */ 1346 1347 if (parser->mark.column != 0) { 1348 parser->mark.column = 0; 1349 parser->mark.line ++; 1350 } 1351 1352 /* Reset the indentation level. */ 1353 1354 if (!yaml_parser_unroll_indent(parser, -1)) 1355 return 0; 1356 1357 /* Reset simple keys. */ 1358 1359 if (!yaml_parser_remove_simple_key(parser)) 1360 return 0; 1361 1362 parser->simple_key_allowed = 0; 1363 1364 /* Create the STREAM-END token and append it to the queue. */ 1365 1366 STREAM_END_TOKEN_INIT(token, parser->mark, parser->mark); 1367 1368 if (!ENQUEUE(parser, parser->tokens, token)) 1369 return 0; 1370 1371 return 1; 1372} 1373 1374/* 1375 * Produce a VERSION-DIRECTIVE or TAG-DIRECTIVE token. 1376 */ 1377 1378static int 1379yaml_parser_fetch_directive(yaml_parser_t *parser) 1380{ 1381 yaml_token_t token; 1382 1383 /* Reset the indentation level. */ 1384 1385 if (!yaml_parser_unroll_indent(parser, -1)) 1386 return 0; 1387 1388 /* Reset simple keys. */ 1389 1390 if (!yaml_parser_remove_simple_key(parser)) 1391 return 0; 1392 1393 parser->simple_key_allowed = 0; 1394 1395 /* Create the YAML-DIRECTIVE or TAG-DIRECTIVE token. */ 1396 1397 if (!yaml_parser_scan_directive(parser, &token)) 1398 return 0; 1399 1400 /* Append the token to the queue. */ 1401 1402 if (!ENQUEUE(parser, parser->tokens, token)) { 1403 yaml_token_delete(&token); 1404 return 0; 1405 } 1406 1407 return 1; 1408} 1409 1410/* 1411 * Produce the DOCUMENT-START or DOCUMENT-END token. 1412 */ 1413 1414static int 1415yaml_parser_fetch_document_indicator(yaml_parser_t *parser, 1416 yaml_token_type_t type) 1417{ 1418 yaml_mark_t start_mark, end_mark; 1419 yaml_token_t token; 1420 1421 /* Reset the indentation level. */ 1422 1423 if (!yaml_parser_unroll_indent(parser, -1)) 1424 return 0; 1425 1426 /* Reset simple keys. */ 1427 1428 if (!yaml_parser_remove_simple_key(parser)) 1429 return 0; 1430 1431 parser->simple_key_allowed = 0; 1432 1433 /* Consume the token. */ 1434 1435 start_mark = parser->mark; 1436 1437 SKIP(parser); 1438 SKIP(parser); 1439 SKIP(parser); 1440 1441 end_mark = parser->mark; 1442 1443 /* Create the DOCUMENT-START or DOCUMENT-END token. */ 1444 1445 TOKEN_INIT(token, type, start_mark, end_mark); 1446 1447 /* Append the token to the queue. */ 1448 1449 if (!ENQUEUE(parser, parser->tokens, token)) 1450 return 0; 1451 1452 return 1; 1453} 1454 1455/* 1456 * Produce the FLOW-SEQUENCE-START or FLOW-MAPPING-START token. 1457 */ 1458 1459static int 1460yaml_parser_fetch_flow_collection_start(yaml_parser_t *parser, 1461 yaml_token_type_t type) 1462{ 1463 yaml_mark_t start_mark, end_mark; 1464 yaml_token_t token; 1465 1466 /* The indicators '[' and '{' may start a simple key. */ 1467 1468 if (!yaml_parser_save_simple_key(parser)) 1469 return 0; 1470 1471 /* Increase the flow level. */ 1472 1473 if (!yaml_parser_increase_flow_level(parser)) 1474 return 0; 1475 1476 /* A simple key may follow the indicators '[' and '{'. */ 1477 1478 parser->simple_key_allowed = 1; 1479 1480 /* Consume the token. */ 1481 1482 start_mark = parser->mark; 1483 SKIP(parser); 1484 end_mark = parser->mark; 1485 1486 /* Create the FLOW-SEQUENCE-START of FLOW-MAPPING-START token. */ 1487 1488 TOKEN_INIT(token, type, start_mark, end_mark); 1489 1490 /* Append the token to the queue. */ 1491 1492 if (!ENQUEUE(parser, parser->tokens, token)) 1493 return 0; 1494 1495 return 1; 1496} 1497 1498/* 1499 * Produce the FLOW-SEQUENCE-END or FLOW-MAPPING-END token. 1500 */ 1501 1502static int 1503yaml_parser_fetch_flow_collection_end(yaml_parser_t *parser, 1504 yaml_token_type_t type) 1505{ 1506 yaml_mark_t start_mark, end_mark; 1507 yaml_token_t token; 1508 1509 /* Reset any potential simple key on the current flow level. */ 1510 1511 if (!yaml_parser_remove_simple_key(parser)) 1512 return 0; 1513 1514 /* Decrease the flow level. */ 1515 1516 if (!yaml_parser_decrease_flow_level(parser)) 1517 return 0; 1518 1519 /* No simple keys after the indicators ']' and '}'. */ 1520 1521 parser->simple_key_allowed = 0; 1522 1523 /* Consume the token. */ 1524 1525 start_mark = parser->mark; 1526 SKIP(parser); 1527 end_mark = parser->mark; 1528 1529 /* Create the FLOW-SEQUENCE-END of FLOW-MAPPING-END token. */ 1530 1531 TOKEN_INIT(token, type, start_mark, end_mark); 1532 1533 /* Append the token to the queue. */ 1534 1535 if (!ENQUEUE(parser, parser->tokens, token)) 1536 return 0; 1537 1538 return 1; 1539} 1540 1541/* 1542 * Produce the FLOW-ENTRY token. 1543 */ 1544 1545static int 1546yaml_parser_fetch_flow_entry(yaml_parser_t *parser) 1547{ 1548 yaml_mark_t start_mark, end_mark; 1549 yaml_token_t token; 1550 1551 /* Reset any potential simple keys on the current flow level. */ 1552 1553 if (!yaml_parser_remove_simple_key(parser)) 1554 return 0; 1555 1556 /* Simple keys are allowed after ','. */ 1557 1558 parser->simple_key_allowed = 1; 1559 1560 /* Consume the token. */ 1561 1562 start_mark = parser->mark; 1563 SKIP(parser); 1564 end_mark = parser->mark; 1565 1566 /* Create the FLOW-ENTRY token and append it to the queue. */ 1567 1568 TOKEN_INIT(token, YAML_FLOW_ENTRY_TOKEN, start_mark, end_mark); 1569 1570 if (!ENQUEUE(parser, parser->tokens, token)) 1571 return 0; 1572 1573 return 1; 1574} 1575 1576/* 1577 * Produce the BLOCK-ENTRY token. 1578 */ 1579 1580static int 1581yaml_parser_fetch_block_entry(yaml_parser_t *parser) 1582{ 1583 yaml_mark_t start_mark, end_mark; 1584 yaml_token_t token; 1585 1586 /* Check if the scanner is in the block context. */ 1587 1588 if (!parser->flow_level) 1589 { 1590 /* Check if we are allowed to start a new entry. */ 1591 1592 if (!parser->simple_key_allowed) { 1593 return yaml_parser_set_scanner_error(parser, NULL, parser->mark, 1594 "block sequence entries are not allowed in this context"); 1595 } 1596 1597 /* Add the BLOCK-SEQUENCE-START token if needed. */ 1598 1599 if (!yaml_parser_roll_indent(parser, parser->mark.column, -1, 1600 YAML_BLOCK_SEQUENCE_START_TOKEN, parser->mark)) 1601 return 0; 1602 } 1603 else 1604 { 1605 /* 1606 * It is an error for the '-' indicator to occur in the flow context, 1607 * but we let the Parser detect and report about it because the Parser 1608 * is able to point to the context. 1609 */ 1610 } 1611 1612 /* Reset any potential simple keys on the current flow level. */ 1613 1614 if (!yaml_parser_remove_simple_key(parser)) 1615 return 0; 1616 1617 /* Simple keys are allowed after '-'. */ 1618 1619 parser->simple_key_allowed = 1; 1620 1621 /* Consume the token. */ 1622 1623 start_mark = parser->mark; 1624 SKIP(parser); 1625 end_mark = parser->mark; 1626 1627 /* Create the BLOCK-ENTRY token and append it to the queue. */ 1628 1629 TOKEN_INIT(token, YAML_BLOCK_ENTRY_TOKEN, start_mark, end_mark); 1630 1631 if (!ENQUEUE(parser, parser->tokens, token)) 1632 return 0; 1633 1634 return 1; 1635} 1636 1637/* 1638 * Produce the KEY token. 1639 */ 1640 1641static int 1642yaml_parser_fetch_key(yaml_parser_t *parser) 1643{ 1644 yaml_mark_t start_mark, end_mark; 1645 yaml_token_t token; 1646 1647 /* In the block context, additional checks are required. */ 1648 1649 if (!parser->flow_level) 1650 { 1651 /* Check if we are allowed to start a new key (not nessesary simple). */ 1652 1653 if (!parser->simple_key_allowed) { 1654 return yaml_parser_set_scanner_error(parser, NULL, parser->mark, 1655 "mapping keys are not allowed in this context"); 1656 } 1657 1658 /* Add the BLOCK-MAPPING-START token if needed. */ 1659 1660 if (!yaml_parser_roll_indent(parser, parser->mark.column, -1, 1661 YAML_BLOCK_MAPPING_START_TOKEN, parser->mark)) 1662 return 0; 1663 } 1664 1665 /* Reset any potential simple keys on the current flow level. */ 1666 1667 if (!yaml_parser_remove_simple_key(parser)) 1668 return 0; 1669 1670 /* Simple keys are allowed after '?' in the block context. */ 1671 1672 parser->simple_key_allowed = (!parser->flow_level); 1673 1674 /* Consume the token. */ 1675 1676 start_mark = parser->mark; 1677 SKIP(parser); 1678 end_mark = parser->mark; 1679 1680 /* Create the KEY token and append it to the queue. */ 1681 1682 TOKEN_INIT(token, YAML_KEY_TOKEN, start_mark, end_mark); 1683 1684 if (!ENQUEUE(parser, parser->tokens, token)) 1685 return 0; 1686 1687 return 1; 1688} 1689 1690/* 1691 * Produce the VALUE token. 1692 */ 1693 1694static int 1695yaml_parser_fetch_value(yaml_parser_t *parser) 1696{ 1697 yaml_mark_t start_mark, end_mark; 1698 yaml_token_t token; 1699 yaml_simple_key_t *simple_key = parser->simple_keys.top-1; 1700 1701 /* Have we found a simple key? */ 1702 1703 if (simple_key->possible) 1704 { 1705 1706 /* Create the KEY token and insert it into the queue. */ 1707 1708 TOKEN_INIT(token, YAML_KEY_TOKEN, simple_key->mark, simple_key->mark); 1709 1710 if (!QUEUE_INSERT(parser, parser->tokens, 1711 simple_key->token_number - parser->tokens_parsed, token)) 1712 return 0; 1713 1714 /* In the block context, we may need to add the BLOCK-MAPPING-START token. */ 1715 1716 if (!yaml_parser_roll_indent(parser, simple_key->mark.column, 1717 simple_key->token_number, 1718 YAML_BLOCK_MAPPING_START_TOKEN, simple_key->mark)) 1719 return 0; 1720 1721 /* Remove the simple key. */ 1722 1723 simple_key->possible = 0; 1724 1725 /* A simple key cannot follow another simple key. */ 1726 1727 parser->simple_key_allowed = 0; 1728 } 1729 else 1730 { 1731 /* The ':' indicator follows a complex key. */ 1732 1733 /* In the block context, extra checks are required. */ 1734 1735 if (!parser->flow_level) 1736 { 1737 /* Check if we are allowed to start a complex value. */ 1738 1739 if (!parser->simple_key_allowed) { 1740 return yaml_parser_set_scanner_error(parser, NULL, parser->mark, 1741 "mapping values are not allowed in this context"); 1742 } 1743 1744 /* Add the BLOCK-MAPPING-START token if needed. */ 1745 1746 if (!yaml_parser_roll_indent(parser, parser->mark.column, -1, 1747 YAML_BLOCK_MAPPING_START_TOKEN, parser->mark)) 1748 return 0; 1749 } 1750 1751 /* Simple keys after ':' are allowed in the block context. */ 1752 1753 parser->simple_key_allowed = (!parser->flow_level); 1754 } 1755 1756 /* Consume the token. */ 1757 1758 start_mark = parser->mark; 1759 SKIP(parser); 1760 end_mark = parser->mark; 1761 1762 /* Create the VALUE token and append it to the queue. */ 1763 1764 TOKEN_INIT(token, YAML_VALUE_TOKEN, start_mark, end_mark); 1765 1766 if (!ENQUEUE(parser, parser->tokens, token)) 1767 return 0; 1768 1769 return 1; 1770} 1771 1772/* 1773 * Produce the ALIAS or ANCHOR token. 1774 */ 1775 1776static int 1777yaml_parser_fetch_anchor(yaml_parser_t *parser, yaml_token_type_t type) 1778{ 1779 yaml_token_t token; 1780 1781 /* An anchor or an alias could be a simple key. */ 1782 1783 if (!yaml_parser_save_simple_key(parser)) 1784 return 0; 1785 1786 /* A simple key cannot follow an anchor or an alias. */ 1787 1788 parser->simple_key_allowed = 0; 1789 1790 /* Cre…
Large files files are truncated, but you can click here to view the full file