PageRenderTime 67ms CodeModel.GetById 12ms app.highlight 44ms RepoModel.GetById 1ms app.codeStats 1ms

/src/libyaml/scanner.c

https://code.google.com/
C | 2455 lines | 1043 code | 552 blank | 860 comment | 247 complexity | c13f0124b3e2df97f153328bd89f1a6e MD5 | raw file

Large files files are truncated, but you can click here to view the full file

   1
   2/*
   3 * Introduction
   4 * ************
   5 *
   6 * The following notes assume that you are familiar with the YAML specification
   7 * (http://yaml.org/spec/cvs/current.html).  We mostly follow it, although in
   8 * some cases we are less restrictive that it requires.
   9 *
  10 * The process of transforming a YAML stream into a sequence of events is
  11 * divided on two steps: Scanning and Parsing.
  12 *
  13 * The Scanner transforms the input stream into a sequence of tokens, while the
  14 * parser transform the sequence of tokens produced by the Scanner into a
  15 * sequence of parsing events.
  16 *
  17 * The Scanner is rather clever and complicated. The Parser, on the contrary,
  18 * is a straightforward implementation of a recursive-descendant parser (or,
  19 * LL(1) parser, as it is usually called).
  20 *
  21 * Actually there are two issues of Scanning that might be called "clever", the
  22 * rest is quite straightforward.  The issues are "block collection start" and
  23 * "simple keys".  Both issues are explained below in details.
  24 *
  25 * Here the Scanning step is explained and implemented.  We start with the list
  26 * of all the tokens produced by the Scanner together with short descriptions.
  27 *
  28 * Now, tokens:
  29 *
  30 *      STREAM-START(encoding)          # The stream start.
  31 *      STREAM-END                      # The stream end.
  32 *      VERSION-DIRECTIVE(major,minor)  # The '%YAML' directive.
  33 *      TAG-DIRECTIVE(handle,prefix)    # The '%TAG' directive.
  34 *      DOCUMENT-START                  # '---'
  35 *      DOCUMENT-END                    # '...'
  36 *      BLOCK-SEQUENCE-START            # Indentation increase denoting a block
  37 *      BLOCK-MAPPING-START             # sequence or a block mapping.
  38 *      BLOCK-END                       # Indentation decrease.
  39 *      FLOW-SEQUENCE-START             # '['
  40 *      FLOW-SEQUENCE-END               # ']'
  41 *      BLOCK-SEQUENCE-START            # '{'
  42 *      BLOCK-SEQUENCE-END              # '}'
  43 *      BLOCK-ENTRY                     # '-'
  44 *      FLOW-ENTRY                      # ','
  45 *      KEY                             # '?' or nothing (simple keys).
  46 *      VALUE                           # ':'
  47 *      ALIAS(anchor)                   # '*anchor'
  48 *      ANCHOR(anchor)                  # '&anchor'
  49 *      TAG(handle,suffix)              # '!handle!suffix'
  50 *      SCALAR(value,style)             # A scalar.
  51 *
  52 * The following two tokens are "virtual" tokens denoting the beginning and the
  53 * end of the stream:
  54 *
  55 *      STREAM-START(encoding)
  56 *      STREAM-END
  57 *
  58 * We pass the information about the input stream encoding with the
  59 * STREAM-START token.
  60 *
  61 * The next two tokens are responsible for tags:
  62 *
  63 *      VERSION-DIRECTIVE(major,minor)
  64 *      TAG-DIRECTIVE(handle,prefix)
  65 *
  66 * Example:
  67 *
  68 *      %YAML   1.1
  69 *      %TAG    !   !foo
  70 *      %TAG    !yaml!  tag:yaml.org,2002:
  71 *      ---
  72 *
  73 * The correspoding sequence of tokens:
  74 *
  75 *      STREAM-START(utf-8)
  76 *      VERSION-DIRECTIVE(1,1)
  77 *      TAG-DIRECTIVE("!","!foo")
  78 *      TAG-DIRECTIVE("!yaml","tag:yaml.org,2002:")
  79 *      DOCUMENT-START
  80 *      STREAM-END
  81 *
  82 * Note that the VERSION-DIRECTIVE and TAG-DIRECTIVE tokens occupy a whole
  83 * line.
  84 *
  85 * The document start and end indicators are represented by:
  86 *
  87 *      DOCUMENT-START
  88 *      DOCUMENT-END
  89 *
  90 * Note that if a YAML stream contains an implicit document (without '---'
  91 * and '...' indicators), no DOCUMENT-START and DOCUMENT-END tokens will be
  92 * produced.
  93 *
  94 * In the following examples, we present whole documents together with the
  95 * produced tokens.
  96 *
  97 *      1. An implicit document:
  98 *
  99 *          'a scalar'
 100 *
 101 *      Tokens:
 102 *
 103 *          STREAM-START(utf-8)
 104 *          SCALAR("a scalar",single-quoted)
 105 *          STREAM-END
 106 *
 107 *      2. An explicit document:
 108 *
 109 *          ---
 110 *          'a scalar'
 111 *          ...
 112 *
 113 *      Tokens:
 114 *
 115 *          STREAM-START(utf-8)
 116 *          DOCUMENT-START
 117 *          SCALAR("a scalar",single-quoted)
 118 *          DOCUMENT-END
 119 *          STREAM-END
 120 *
 121 *      3. Several documents in a stream:
 122 *
 123 *          'a scalar'
 124 *          ---
 125 *          'another scalar'
 126 *          ---
 127 *          'yet another scalar'
 128 *
 129 *      Tokens:
 130 *
 131 *          STREAM-START(utf-8)
 132 *          SCALAR("a scalar",single-quoted)
 133 *          DOCUMENT-START
 134 *          SCALAR("another scalar",single-quoted)
 135 *          DOCUMENT-START
 136 *          SCALAR("yet another scalar",single-quoted)
 137 *          STREAM-END
 138 *
 139 * We have already introduced the SCALAR token above.  The following tokens are
 140 * used to describe aliases, anchors, tag, and scalars:
 141 *
 142 *      ALIAS(anchor)
 143 *      ANCHOR(anchor)
 144 *      TAG(handle,suffix)
 145 *      SCALAR(value,style)
 146 *
 147 * The following series of examples illustrate the usage of these tokens:
 148 *
 149 *      1. A recursive sequence:
 150 *
 151 *          &A [ *A ]
 152 *
 153 *      Tokens:
 154 *
 155 *          STREAM-START(utf-8)
 156 *          ANCHOR("A")
 157 *          FLOW-SEQUENCE-START
 158 *          ALIAS("A")
 159 *          FLOW-SEQUENCE-END
 160 *          STREAM-END
 161 *
 162 *      2. A tagged scalar:
 163 *
 164 *          !!float "3.14"  # A good approximation.
 165 *
 166 *      Tokens:
 167 *
 168 *          STREAM-START(utf-8)
 169 *          TAG("!!","float")
 170 *          SCALAR("3.14",double-quoted)
 171 *          STREAM-END
 172 *
 173 *      3. Various scalar styles:
 174 *
 175 *          --- # Implicit empty plain scalars do not produce tokens.
 176 *          --- a plain scalar
 177 *          --- 'a single-quoted scalar'
 178 *          --- "a double-quoted scalar"
 179 *          --- |-
 180 *            a literal scalar
 181 *          --- >-
 182 *            a folded
 183 *            scalar
 184 *
 185 *      Tokens:
 186 *
 187 *          STREAM-START(utf-8)
 188 *          DOCUMENT-START
 189 *          DOCUMENT-START
 190 *          SCALAR("a plain scalar",plain)
 191 *          DOCUMENT-START
 192 *          SCALAR("a single-quoted scalar",single-quoted)
 193 *          DOCUMENT-START
 194 *          SCALAR("a double-quoted scalar",double-quoted)
 195 *          DOCUMENT-START
 196 *          SCALAR("a literal scalar",literal)
 197 *          DOCUMENT-START
 198 *          SCALAR("a folded scalar",folded)
 199 *          STREAM-END
 200 *
 201 * Now it's time to review collection-related tokens. We will start with
 202 * flow collections:
 203 *
 204 *      FLOW-SEQUENCE-START
 205 *      FLOW-SEQUENCE-END
 206 *      FLOW-MAPPING-START
 207 *      FLOW-MAPPING-END
 208 *      FLOW-ENTRY
 209 *      KEY
 210 *      VALUE
 211 *
 212 * The tokens FLOW-SEQUENCE-START, FLOW-SEQUENCE-END, FLOW-MAPPING-START, and
 213 * FLOW-MAPPING-END represent the indicators '[', ']', '{', and '}'
 214 * correspondingly.  FLOW-ENTRY represent the ',' indicator.  Finally the
 215 * indicators '?' and ':', which are used for denoting mapping keys and values,
 216 * are represented by the KEY and VALUE tokens.
 217 *
 218 * The following examples show flow collections:
 219 *
 220 *      1. A flow sequence:
 221 *
 222 *          [item 1, item 2, item 3]
 223 *
 224 *      Tokens:
 225 *
 226 *          STREAM-START(utf-8)
 227 *          FLOW-SEQUENCE-START
 228 *          SCALAR("item 1",plain)
 229 *          FLOW-ENTRY
 230 *          SCALAR("item 2",plain)
 231 *          FLOW-ENTRY
 232 *          SCALAR("item 3",plain)
 233 *          FLOW-SEQUENCE-END
 234 *          STREAM-END
 235 *
 236 *      2. A flow mapping:
 237 *
 238 *          {
 239 *              a simple key: a value,  # Note that the KEY token is produced.
 240 *              ? a complex key: another value,
 241 *          }
 242 *
 243 *      Tokens:
 244 *
 245 *          STREAM-START(utf-8)
 246 *          FLOW-MAPPING-START
 247 *          KEY
 248 *          SCALAR("a simple key",plain)
 249 *          VALUE
 250 *          SCALAR("a value",plain)
 251 *          FLOW-ENTRY
 252 *          KEY
 253 *          SCALAR("a complex key",plain)
 254 *          VALUE
 255 *          SCALAR("another value",plain)
 256 *          FLOW-ENTRY
 257 *          FLOW-MAPPING-END
 258 *          STREAM-END
 259 *
 260 * A simple key is a key which is not denoted by the '?' indicator.  Note that
 261 * the Scanner still produce the KEY token whenever it encounters a simple key.
 262 *
 263 * For scanning block collections, the following tokens are used (note that we
 264 * repeat KEY and VALUE here):
 265 *
 266 *      BLOCK-SEQUENCE-START
 267 *      BLOCK-MAPPING-START
 268 *      BLOCK-END
 269 *      BLOCK-ENTRY
 270 *      KEY
 271 *      VALUE
 272 *
 273 * The tokens BLOCK-SEQUENCE-START and BLOCK-MAPPING-START denote indentation
 274 * increase that precedes a block collection (cf. the INDENT token in Python).
 275 * The token BLOCK-END denote indentation decrease that ends a block collection
 276 * (cf. the DEDENT token in Python).  However YAML has some syntax pecularities
 277 * that makes detections of these tokens more complex.
 278 *
 279 * The tokens BLOCK-ENTRY, KEY, and VALUE are used to represent the indicators
 280 * '-', '?', and ':' correspondingly.
 281 *
 282 * The following examples show how the tokens BLOCK-SEQUENCE-START,
 283 * BLOCK-MAPPING-START, and BLOCK-END are emitted by the Scanner:
 284 *
 285 *      1. Block sequences:
 286 *
 287 *          - item 1
 288 *          - item 2
 289 *          -
 290 *            - item 3.1
 291 *            - item 3.2
 292 *          -
 293 *            key 1: value 1
 294 *            key 2: value 2
 295 *
 296 *      Tokens:
 297 *
 298 *          STREAM-START(utf-8)
 299 *          BLOCK-SEQUENCE-START
 300 *          BLOCK-ENTRY
 301 *          SCALAR("item 1",plain)
 302 *          BLOCK-ENTRY
 303 *          SCALAR("item 2",plain)
 304 *          BLOCK-ENTRY
 305 *          BLOCK-SEQUENCE-START
 306 *          BLOCK-ENTRY
 307 *          SCALAR("item 3.1",plain)
 308 *          BLOCK-ENTRY
 309 *          SCALAR("item 3.2",plain)
 310 *          BLOCK-END
 311 *          BLOCK-ENTRY
 312 *          BLOCK-MAPPING-START
 313 *          KEY
 314 *          SCALAR("key 1",plain)
 315 *          VALUE
 316 *          SCALAR("value 1",plain)
 317 *          KEY
 318 *          SCALAR("key 2",plain)
 319 *          VALUE
 320 *          SCALAR("value 2",plain)
 321 *          BLOCK-END
 322 *          BLOCK-END
 323 *          STREAM-END
 324 *
 325 *      2. Block mappings:
 326 *
 327 *          a simple key: a value   # The KEY token is produced here.
 328 *          ? a complex key
 329 *          : another value
 330 *          a mapping:
 331 *            key 1: value 1
 332 *            key 2: value 2
 333 *          a sequence:
 334 *            - item 1
 335 *            - item 2
 336 *
 337 *      Tokens:
 338 *
 339 *          STREAM-START(utf-8)
 340 *          BLOCK-MAPPING-START
 341 *          KEY
 342 *          SCALAR("a simple key",plain)
 343 *          VALUE
 344 *          SCALAR("a value",plain)
 345 *          KEY
 346 *          SCALAR("a complex key",plain)
 347 *          VALUE
 348 *          SCALAR("another value",plain)
 349 *          KEY
 350 *          SCALAR("a mapping",plain)
 351 *          BLOCK-MAPPING-START
 352 *          KEY
 353 *          SCALAR("key 1",plain)
 354 *          VALUE
 355 *          SCALAR("value 1",plain)
 356 *          KEY
 357 *          SCALAR("key 2",plain)
 358 *          VALUE
 359 *          SCALAR("value 2",plain)
 360 *          BLOCK-END
 361 *          KEY
 362 *          SCALAR("a sequence",plain)
 363 *          VALUE
 364 *          BLOCK-SEQUENCE-START
 365 *          BLOCK-ENTRY
 366 *          SCALAR("item 1",plain)
 367 *          BLOCK-ENTRY
 368 *          SCALAR("item 2",plain)
 369 *          BLOCK-END
 370 *          BLOCK-END
 371 *          STREAM-END
 372 *
 373 * YAML does not always require to start a new block collection from a new
 374 * line.  If the current line contains only '-', '?', and ':' indicators, a new
 375 * block collection may start at the current line.  The following examples
 376 * illustrate this case:
 377 *
 378 *      1. Collections in a sequence:
 379 *
 380 *          - - item 1
 381 *            - item 2
 382 *          - key 1: value 1
 383 *            key 2: value 2
 384 *          - ? complex key
 385 *            : complex value
 386 *
 387 *      Tokens:
 388 *
 389 *          STREAM-START(utf-8)
 390 *          BLOCK-SEQUENCE-START
 391 *          BLOCK-ENTRY
 392 *          BLOCK-SEQUENCE-START
 393 *          BLOCK-ENTRY
 394 *          SCALAR("item 1",plain)
 395 *          BLOCK-ENTRY
 396 *          SCALAR("item 2",plain)
 397 *          BLOCK-END
 398 *          BLOCK-ENTRY
 399 *          BLOCK-MAPPING-START
 400 *          KEY
 401 *          SCALAR("key 1",plain)
 402 *          VALUE
 403 *          SCALAR("value 1",plain)
 404 *          KEY
 405 *          SCALAR("key 2",plain)
 406 *          VALUE
 407 *          SCALAR("value 2",plain)
 408 *          BLOCK-END
 409 *          BLOCK-ENTRY
 410 *          BLOCK-MAPPING-START
 411 *          KEY
 412 *          SCALAR("complex key")
 413 *          VALUE
 414 *          SCALAR("complex value")
 415 *          BLOCK-END
 416 *          BLOCK-END
 417 *          STREAM-END
 418 *
 419 *      2. Collections in a mapping:
 420 *
 421 *          ? a sequence
 422 *          : - item 1
 423 *            - item 2
 424 *          ? a mapping
 425 *          : key 1: value 1
 426 *            key 2: value 2
 427 *
 428 *      Tokens:
 429 *
 430 *          STREAM-START(utf-8)
 431 *          BLOCK-MAPPING-START
 432 *          KEY
 433 *          SCALAR("a sequence",plain)
 434 *          VALUE
 435 *          BLOCK-SEQUENCE-START
 436 *          BLOCK-ENTRY
 437 *          SCALAR("item 1",plain)
 438 *          BLOCK-ENTRY
 439 *          SCALAR("item 2",plain)
 440 *          BLOCK-END
 441 *          KEY
 442 *          SCALAR("a mapping",plain)
 443 *          VALUE
 444 *          BLOCK-MAPPING-START
 445 *          KEY
 446 *          SCALAR("key 1",plain)
 447 *          VALUE
 448 *          SCALAR("value 1",plain)
 449 *          KEY
 450 *          SCALAR("key 2",plain)
 451 *          VALUE
 452 *          SCALAR("value 2",plain)
 453 *          BLOCK-END
 454 *          BLOCK-END
 455 *          STREAM-END
 456 *
 457 * YAML also permits non-indented sequences if they are included into a block
 458 * mapping.  In this case, the token BLOCK-SEQUENCE-START is not produced:
 459 *
 460 *      key:
 461 *      - item 1    # BLOCK-SEQUENCE-START is NOT produced here.
 462 *      - item 2
 463 *
 464 * Tokens:
 465 *
 466 *      STREAM-START(utf-8)
 467 *      BLOCK-MAPPING-START
 468 *      KEY
 469 *      SCALAR("key",plain)
 470 *      VALUE
 471 *      BLOCK-ENTRY
 472 *      SCALAR("item 1",plain)
 473 *      BLOCK-ENTRY
 474 *      SCALAR("item 2",plain)
 475 *      BLOCK-END
 476 */
 477
 478#include "yaml_private.h"
 479
 480/*
 481 * Ensure that the buffer contains the required number of characters.
 482 * Return 1 on success, 0 on failure (reader error or memory error).
 483 */
 484
 485#define CACHE(parser,length)                                                    \
 486    (parser->unread >= (length)                                                 \
 487        ? 1                                                                     \
 488        : yaml_parser_update_buffer(parser, (length)))
 489
 490/*
 491 * Advance the buffer pointer.
 492 */
 493
 494#define SKIP(parser)                                                            \
 495     (parser->mark.index ++,                                                    \
 496      parser->mark.column ++,                                                   \
 497      parser->unread --,                                                        \
 498      parser->buffer.pointer ++)
 499
 500#define SKIPN(parser,n)                                                         \
 501     (parser->mark.index ++,                                                    \
 502      parser->mark.column ++,                                                   \
 503      parser->unread --,                                                        \
 504      parser->buffer.pointer += (n))
 505#define SKIP_LINE(parser)                                                       \
 506     (IS_CRLF(parser->buffer) ?                                                 \
 507      (parser->mark.index += 2,                                                 \
 508       parser->mark.column = 0,                                                 \
 509       parser->mark.line ++,                                                    \
 510       parser->unread -= 2,                                                     \
 511       parser->buffer.pointer += 2) :                                           \
 512      IS_BREAK(parser->buffer) ?                                                \
 513      (parser->mark.index ++,                                                   \
 514       parser->mark.column = 0,                                                 \
 515       parser->mark.line ++,                                                    \
 516       parser->unread --,                                                       \
 517       parser->buffer.pointer += WIDTH(parser->buffer)) : 0)
 518
 519/*
 520 * Copy a character to a string buffer and advance pointers.
 521 */
 522
 523#define READ(parser,string)                                                     \
 524     (STRING_EXTEND(parser,string) ?                                            \
 525         (COPY(string,parser->buffer),                                          \
 526          parser->mark.index ++,                                                \
 527          parser->mark.column ++,                                               \
 528          parser->unread --,                                                    \
 529          1) : 0)
 530
 531#define READN(parser,string,n)                                                  \
 532     (STRING_EXTEND(parser,string) ?                                            \
 533         (COPYN(string,parser->buffer,n),                                       \
 534          parser->mark.index ++,                                                \
 535          parser->mark.column ++,                                               \
 536          parser->unread --,                                                    \
 537          1) : 0)
 538
 539/*
 540 * Copy a line break character to a string buffer and advance pointers.
 541 */
 542
 543#define READ_LINE(parser,string)                                                \
 544    (STRING_EXTEND(parser,string) ?                                             \
 545    (((CHECK_AT(parser->buffer,'\r',0)                                          \
 546       && CHECK_AT(parser->buffer,'\n',1)) ?        /* CR LF -> LF */           \
 547     (*((string).pointer++) = (yaml_char_t) '\n',                               \
 548      parser->buffer.pointer += 2,                                              \
 549      parser->mark.index += 2,                                                  \
 550      parser->mark.column = 0,                                                  \
 551      parser->mark.line ++,                                                     \
 552      parser->unread -= 2) :                                                    \
 553     (CHECK_AT(parser->buffer,'\r',0)                                           \
 554      || CHECK_AT(parser->buffer,'\n',0)) ?         /* CR|LF -> LF */           \
 555     (*((string).pointer++) = (yaml_char_t) '\n',                               \
 556      parser->buffer.pointer ++,                                                \
 557      parser->mark.index ++,                                                    \
 558      parser->mark.column = 0,                                                  \
 559      parser->mark.line ++,                                                     \
 560      parser->unread --) :                                                      \
 561     (CHECK_AT(parser->buffer,'\xC2',0)                                         \
 562      && CHECK_AT(parser->buffer,'\x85',1)) ?       /* NEL -> LF */             \
 563     (*((string).pointer++) = (yaml_char_t) '\n',                               \
 564      parser->buffer.pointer += 2,                                              \
 565      parser->mark.index ++,                                                    \
 566      parser->mark.column = 0,                                                  \
 567      parser->mark.line ++,                                                     \
 568      parser->unread --) :                                                      \
 569     (CHECK_AT(parser->buffer,'\xE2',0) &&                                      \
 570      CHECK_AT(parser->buffer,'\x80',1) &&                                      \
 571      (CHECK_AT(parser->buffer,'\xA8',2) ||                                     \
 572       CHECK_AT(parser->buffer,'\xA9',2))) ?        /* LS|PS -> LS|PS */        \
 573     (*((string).pointer++) = *(parser->buffer.pointer++),                      \
 574      *((string).pointer++) = *(parser->buffer.pointer++),                      \
 575      *((string).pointer++) = *(parser->buffer.pointer++),                      \
 576      parser->mark.index ++,                                                    \
 577      parser->mark.column = 0,                                                  \
 578      parser->mark.line ++,                                                     \
 579      parser->unread --) : 0),                                                  \
 580    1) : 0)
 581
 582/*
 583 * Public API declarations.
 584 */
 585
 586YAML_DECLARE(int)
 587yaml_parser_scan(yaml_parser_t *parser, yaml_token_t *token);
 588
 589/*
 590 * Error handling.
 591 */
 592
 593static int
 594yaml_parser_set_scanner_error(yaml_parser_t *parser, const char *context,
 595        yaml_mark_t context_mark, const char *problem);
 596
 597/*
 598 * High-level token API.
 599 */
 600
 601YAML_DECLARE(int)
 602yaml_parser_fetch_more_tokens(yaml_parser_t *parser);
 603
 604static int
 605yaml_parser_fetch_next_token(yaml_parser_t *parser);
 606
 607/*
 608 * Potential simple keys.
 609 */
 610
 611static int
 612yaml_parser_stale_simple_keys(yaml_parser_t *parser);
 613
 614static int
 615yaml_parser_save_simple_key(yaml_parser_t *parser);
 616
 617static int
 618yaml_parser_remove_simple_key(yaml_parser_t *parser);
 619
 620static int
 621yaml_parser_increase_flow_level(yaml_parser_t *parser);
 622
 623static int
 624yaml_parser_decrease_flow_level(yaml_parser_t *parser);
 625
 626/*
 627 * Indentation treatment.
 628 */
 629
 630static int
 631yaml_parser_roll_indent(yaml_parser_t *parser, int column,
 632        int number, yaml_token_type_t type, yaml_mark_t mark);
 633
 634static int
 635yaml_parser_unroll_indent(yaml_parser_t *parser, int column);
 636
 637/*
 638 * Token fetchers.
 639 */
 640
 641static int
 642yaml_parser_fetch_stream_start(yaml_parser_t *parser);
 643
 644static int
 645yaml_parser_fetch_stream_end(yaml_parser_t *parser);
 646
 647static int
 648yaml_parser_fetch_directive(yaml_parser_t *parser);
 649
 650static int
 651yaml_parser_fetch_document_indicator(yaml_parser_t *parser,
 652        yaml_token_type_t type);
 653
 654static int
 655yaml_parser_fetch_flow_collection_start(yaml_parser_t *parser,
 656        yaml_token_type_t type);
 657
 658static int
 659yaml_parser_fetch_flow_collection_end(yaml_parser_t *parser,
 660        yaml_token_type_t type);
 661
 662static int
 663yaml_parser_fetch_flow_entry(yaml_parser_t *parser);
 664
 665static int
 666yaml_parser_fetch_block_entry(yaml_parser_t *parser);
 667
 668static int
 669yaml_parser_fetch_key(yaml_parser_t *parser);
 670
 671static int
 672yaml_parser_fetch_value(yaml_parser_t *parser);
 673
 674static int
 675yaml_parser_fetch_anchor(yaml_parser_t *parser, yaml_token_type_t type);
 676
 677static int
 678yaml_parser_fetch_tag(yaml_parser_t *parser);
 679
 680static int
 681yaml_parser_fetch_block_scalar(yaml_parser_t *parser, int literal);
 682
 683static int
 684yaml_parser_fetch_flow_scalar(yaml_parser_t *parser, int single);
 685
 686static int
 687yaml_parser_fetch_plain_scalar(yaml_parser_t *parser);
 688
 689/*
 690 * Token scanners.
 691 */
 692
 693static int
 694yaml_parser_scan_to_next_token(yaml_parser_t *parser);
 695
 696static int
 697yaml_parser_scan_directive(yaml_parser_t *parser, yaml_token_t *token);
 698
 699static int
 700yaml_parser_scan_directive_name(yaml_parser_t *parser,
 701        yaml_mark_t start_mark, yaml_char_t **name);
 702
 703static int
 704yaml_parser_scan_version_directive_value(yaml_parser_t *parser,
 705        yaml_mark_t start_mark, int *major, int *minor);
 706
 707static int
 708yaml_parser_scan_version_directive_number(yaml_parser_t *parser,
 709        yaml_mark_t start_mark, int *number);
 710
 711static int
 712yaml_parser_scan_tag_directive_value(yaml_parser_t *parser,
 713        yaml_mark_t mark, yaml_char_t **handle, yaml_char_t **prefix);
 714
 715static int
 716yaml_parser_scan_anchor(yaml_parser_t *parser, yaml_token_t *token,
 717        yaml_token_type_t type);
 718
 719static int
 720yaml_parser_scan_tag(yaml_parser_t *parser, yaml_token_t *token);
 721
 722static int
 723yaml_parser_scan_tag_handle(yaml_parser_t *parser, int directive,
 724        yaml_mark_t start_mark, yaml_char_t **handle);
 725
 726static int
 727yaml_parser_scan_tag_uri(yaml_parser_t *parser, int directive,
 728        yaml_char_t *head, yaml_mark_t start_mark, yaml_char_t **uri);
 729
 730static int
 731yaml_parser_scan_uri_escapes(yaml_parser_t *parser, int directive,
 732        yaml_mark_t start_mark, yaml_string_t *string);
 733
 734static int
 735yaml_parser_scan_block_scalar(yaml_parser_t *parser, yaml_token_t *token,
 736        int literal);
 737
 738static int
 739yaml_parser_scan_block_scalar_breaks(yaml_parser_t *parser,
 740        int *indent, yaml_string_t *breaks,
 741        yaml_mark_t start_mark, yaml_mark_t *end_mark);
 742
 743static int
 744yaml_parser_scan_flow_scalar(yaml_parser_t *parser, yaml_token_t *token,
 745        int single);
 746
 747static int
 748yaml_parser_scan_plain_scalar(yaml_parser_t *parser, yaml_token_t *token);
 749
 750/*
 751 * Get the next token.
 752 */
 753
 754YAML_DECLARE(int)
 755yaml_parser_scan(yaml_parser_t *parser, yaml_token_t *token)
 756{
 757    assert(parser); /* Non-NULL parser object is expected. */
 758    assert(token);  /* Non-NULL token object is expected. */
 759
 760    /* Erase the token object. */
 761
 762    memset(token, 0, sizeof(yaml_token_t));
 763
 764    /* No tokens after STREAM-END or error. */
 765
 766    if (parser->stream_end_produced || parser->error) {
 767        return 1;
 768    }
 769
 770    /* Ensure that the tokens queue contains enough tokens. */
 771
 772    if (!parser->token_available) {
 773        if (!yaml_parser_fetch_more_tokens(parser))
 774            return 0;
 775    }
 776
 777    /* Fetch the next token from the queue. */
 778    
 779    *token = DEQUEUE(parser, parser->tokens);
 780    parser->token_available = 0;
 781    parser->tokens_parsed ++;
 782
 783    if (token->type == YAML_STREAM_END_TOKEN) {
 784        parser->stream_end_produced = 1;
 785    }
 786
 787    return 1;
 788}
 789
 790/*
 791 * Set the scanner error and return 0.
 792 */
 793
 794static int
 795yaml_parser_set_scanner_error(yaml_parser_t *parser, const char *context,
 796        yaml_mark_t context_mark, const char *problem)
 797{
 798    parser->error = YAML_SCANNER_ERROR;
 799    parser->context = context;
 800    parser->context_mark = context_mark;
 801    parser->problem = problem;
 802    parser->problem_mark = parser->mark;
 803
 804    return 0;
 805}
 806
 807/*
 808 * Ensure that the tokens queue contains at least one token which can be
 809 * returned to the Parser.
 810 */
 811
 812YAML_DECLARE(int)
 813yaml_parser_fetch_more_tokens(yaml_parser_t *parser)
 814{
 815    int need_more_tokens;
 816
 817    /* While we need more tokens to fetch, do it. */
 818
 819    while (1)
 820    {
 821        /*
 822         * Check if we really need to fetch more tokens.
 823         */
 824
 825        need_more_tokens = 0;
 826
 827        if (parser->tokens.head == parser->tokens.tail)
 828        {
 829            /* Queue is empty. */
 830
 831            need_more_tokens = 1;
 832        }
 833        else
 834        {
 835            yaml_simple_key_t *simple_key;
 836
 837            /* Check if any potential simple key may occupy the head position. */
 838
 839            if (!yaml_parser_stale_simple_keys(parser))
 840                return 0;
 841
 842            for (simple_key = parser->simple_keys.start;
 843                    simple_key != parser->simple_keys.top; simple_key++) {
 844                if (simple_key->possible
 845                        && simple_key->token_number == parser->tokens_parsed) {
 846                    need_more_tokens = 1;
 847                    break;
 848                }
 849            }
 850        }
 851
 852        /* We are finished. */
 853
 854        if (!need_more_tokens)
 855            break;
 856
 857        /* Fetch the next token. */
 858
 859        if (!yaml_parser_fetch_next_token(parser))
 860            return 0;
 861    }
 862
 863    parser->token_available = 1;
 864
 865    return 1;
 866}
 867
 868/*
 869 * The dispatcher for token fetchers.
 870 */
 871
 872static int
 873yaml_parser_fetch_next_token(yaml_parser_t *parser)
 874{
 875    /* Ensure that the buffer is initialized. */
 876
 877    if (!CACHE(parser, 1))
 878        return 0;
 879
 880    /* Check if we just started scanning.  Fetch STREAM-START then. */
 881
 882    if (!parser->stream_start_produced)
 883        return yaml_parser_fetch_stream_start(parser);
 884
 885    /* Eat whitespaces and comments until we reach the next token. */
 886
 887    if (!yaml_parser_scan_to_next_token(parser))
 888        return 0;
 889
 890    /* Remove obsolete potential simple keys. */
 891
 892    if (!yaml_parser_stale_simple_keys(parser))
 893        return 0;
 894
 895    /* Check the indentation level against the current column. */
 896
 897    if (!yaml_parser_unroll_indent(parser, parser->mark.column))
 898        return 0;
 899
 900    /*
 901     * Ensure that the buffer contains at least 4 characters.  4 is the length
 902     * of the longest indicators ('--- ' and '... ').
 903     */
 904
 905    if (!CACHE(parser, 4))
 906        return 0;
 907
 908    /* Is it the end of the stream? */
 909
 910    if (IS_Z(parser->buffer))
 911        return yaml_parser_fetch_stream_end(parser);
 912
 913    /* Is it a directive? */
 914
 915    if (parser->mark.column == 0 && CHECK(parser->buffer, '%'))
 916        return yaml_parser_fetch_directive(parser);
 917
 918    /* Is it the document start indicator? */
 919
 920    if (parser->mark.column == 0
 921            && CHECK_AT(parser->buffer, '-', 0)
 922            && CHECK_AT(parser->buffer, '-', 1)
 923            && CHECK_AT(parser->buffer, '-', 2)
 924            && IS_BLANKZ_AT(parser->buffer, 3))
 925        return yaml_parser_fetch_document_indicator(parser,
 926                YAML_DOCUMENT_START_TOKEN);
 927
 928    /* Is it the document end indicator? */
 929
 930    if (parser->mark.column == 0
 931            && CHECK_AT(parser->buffer, '.', 0)
 932            && CHECK_AT(parser->buffer, '.', 1)
 933            && CHECK_AT(parser->buffer, '.', 2)
 934            && IS_BLANKZ_AT(parser->buffer, 3))
 935        return yaml_parser_fetch_document_indicator(parser,
 936                YAML_DOCUMENT_END_TOKEN);
 937
 938    /* Is it the flow sequence start indicator? */
 939
 940    if (CHECK(parser->buffer, '['))
 941        return yaml_parser_fetch_flow_collection_start(parser,
 942                YAML_FLOW_SEQUENCE_START_TOKEN);
 943
 944    /* Is it the flow mapping start indicator? */
 945
 946    if (CHECK(parser->buffer, '{'))
 947        return yaml_parser_fetch_flow_collection_start(parser,
 948                YAML_FLOW_MAPPING_START_TOKEN);
 949
 950    /* Is it the flow sequence end indicator? */
 951
 952    if (CHECK(parser->buffer, ']'))
 953        return yaml_parser_fetch_flow_collection_end(parser,
 954                YAML_FLOW_SEQUENCE_END_TOKEN);
 955
 956    /* Is it the flow mapping end indicator? */
 957
 958    if (CHECK(parser->buffer, '}'))
 959        return yaml_parser_fetch_flow_collection_end(parser,
 960                YAML_FLOW_MAPPING_END_TOKEN);
 961
 962    /* Is it the flow entry indicator? */
 963
 964    if (CHECK(parser->buffer, ','))
 965        return yaml_parser_fetch_flow_entry(parser);
 966
 967    /* Is it the block entry indicator? */
 968
 969    if (CHECK(parser->buffer, '-') && IS_BLANKZ_AT(parser->buffer, 1))
 970        return yaml_parser_fetch_block_entry(parser);
 971
 972    /* Is it the key indicator? */
 973
 974    if (CHECK(parser->buffer, '?')
 975            && (parser->flow_level || IS_BLANKZ_AT(parser->buffer, 1)))
 976        return yaml_parser_fetch_key(parser);
 977
 978    /* Is it the value indicator? */
 979
 980    if (CHECK(parser->buffer, ':')
 981            && (parser->flow_level || IS_BLANKZ_AT(parser->buffer, 1)))
 982        return yaml_parser_fetch_value(parser);
 983
 984    /* Is it an alias? */
 985
 986    if (CHECK(parser->buffer, '*'))
 987        return yaml_parser_fetch_anchor(parser, YAML_ALIAS_TOKEN);
 988
 989    /* Is it an anchor? */
 990
 991    if (CHECK(parser->buffer, '&'))
 992        return yaml_parser_fetch_anchor(parser, YAML_ANCHOR_TOKEN);
 993
 994    /* Is it a tag? */
 995
 996    if (CHECK(parser->buffer, '!'))
 997        return yaml_parser_fetch_tag(parser);
 998
 999    /* Is it a literal scalar? */
1000
1001    if (CHECK(parser->buffer, '|') && !parser->flow_level)
1002        return yaml_parser_fetch_block_scalar(parser, 1);
1003
1004    /* Is it a folded scalar? */
1005
1006    if (CHECK(parser->buffer, '>') && !parser->flow_level)
1007        return yaml_parser_fetch_block_scalar(parser, 0);
1008
1009    /* Is it a single-quoted scalar? */
1010
1011    if (CHECK(parser->buffer, '\''))
1012        return yaml_parser_fetch_flow_scalar(parser, 1);
1013
1014    /* Is it a double-quoted scalar? */
1015
1016    if (CHECK(parser->buffer, '"'))
1017        return yaml_parser_fetch_flow_scalar(parser, 0);
1018
1019    /*
1020     * Is it a plain scalar?
1021     *
1022     * A plain scalar may start with any non-blank characters except
1023     *
1024     *      '-', '?', ':', ',', '[', ']', '{', '}',
1025     *      '#', '&', '*', '!', '|', '>', '\'', '\"',
1026     *      '%', '@', '`'.
1027     *
1028     * In the block context (and, for the '-' indicator, in the flow context
1029     * too), it may also start with the characters
1030     *
1031     *      '-', '?', ':'
1032     *
1033     * if it is followed by a non-space character.
1034     *
1035     * The last rule is more restrictive than the specification requires.
1036     */
1037
1038    if (!(IS_BLANKZ(parser->buffer) || CHECK(parser->buffer, '-')
1039                || CHECK(parser->buffer, '?') || CHECK(parser->buffer, ':')
1040                || CHECK(parser->buffer, ',') || CHECK(parser->buffer, '[')
1041                || CHECK(parser->buffer, ']') || CHECK(parser->buffer, '{')
1042                || CHECK(parser->buffer, '}') || CHECK(parser->buffer, '#')
1043                || CHECK(parser->buffer, '&') || CHECK(parser->buffer, '*')
1044                || CHECK(parser->buffer, '!') || CHECK(parser->buffer, '|')
1045                || CHECK(parser->buffer, '>') || CHECK(parser->buffer, '\'')
1046                || CHECK(parser->buffer, '"') || CHECK(parser->buffer, '%')
1047                || CHECK(parser->buffer, '@') || CHECK(parser->buffer, '`')) ||
1048            (CHECK(parser->buffer, '-') && !IS_BLANK_AT(parser->buffer, 1)) ||
1049            (!parser->flow_level &&
1050             (CHECK(parser->buffer, '?') || CHECK(parser->buffer, ':'))
1051             && !IS_BLANKZ_AT(parser->buffer, 1)))
1052        return yaml_parser_fetch_plain_scalar(parser);
1053
1054    /*
1055     * If we don't determine the token type so far, it is an error.
1056     */
1057
1058    return yaml_parser_set_scanner_error(parser,
1059            "while scanning for the next token", parser->mark,
1060            "found character that cannot start any token");
1061}
1062
1063/*
1064 * Check the list of potential simple keys and remove the positions that
1065 * cannot contain simple keys anymore.
1066 */
1067
1068static int
1069yaml_parser_stale_simple_keys(yaml_parser_t *parser)
1070{
1071    yaml_simple_key_t *simple_key;
1072
1073    /* Check for a potential simple key for each flow level. */
1074
1075    for (simple_key = parser->simple_keys.start;
1076            simple_key != parser->simple_keys.top; simple_key ++)
1077    {
1078        /*
1079         * The specification requires that a simple key
1080         *
1081         *  - is limited to a single line,
1082         *  - is shorter than 1024 characters.
1083         */
1084
1085        if (simple_key->possible
1086                && (simple_key->mark.line < parser->mark.line
1087                    || simple_key->mark.index+1024 < parser->mark.index)) {
1088
1089            /* Check if the potential simple key to be removed is required. */
1090
1091            if (simple_key->required) {
1092                return yaml_parser_set_scanner_error(parser,
1093                        "while scanning a simple key", simple_key->mark,
1094                        "could not find expected ':'");
1095            }
1096
1097            simple_key->possible = 0;
1098        }
1099    }
1100
1101    return 1;
1102}
1103
1104/*
1105 * Check if a simple key may start at the current position and add it if
1106 * needed.
1107 */
1108
1109static int
1110yaml_parser_save_simple_key(yaml_parser_t *parser)
1111{
1112    /*
1113     * A simple key is required at the current position if the scanner is in
1114     * the block context and the current column coincides with the indentation
1115     * level.
1116     */
1117
1118    int required = (!parser->flow_level
1119            && parser->indent == (int)parser->mark.column);
1120
1121    /*
1122     * A simple key is required only when it is the first token in the current
1123     * line.  Therefore it is always allowed.  But we add a check anyway.
1124     */
1125
1126    assert(parser->simple_key_allowed || !required);    /* Impossible. */
1127
1128    /*
1129     * If the current position may start a simple key, save it.
1130     */
1131
1132    if (parser->simple_key_allowed)
1133    {
1134        yaml_simple_key_t simple_key;
1135        simple_key.possible = 1;
1136        simple_key.required = required;
1137        simple_key.token_number = 
1138            parser->tokens_parsed + parser->tokens.tail - parser->tokens.head;
1139        simple_key.mark = parser->mark;
1140
1141        if (!yaml_parser_remove_simple_key(parser)) return 0;
1142
1143        *(parser->simple_keys.top-1) = simple_key;
1144    }
1145
1146    return 1;
1147}
1148
1149/*
1150 * Remove a potential simple key at the current flow level.
1151 */
1152
1153static int
1154yaml_parser_remove_simple_key(yaml_parser_t *parser)
1155{
1156    yaml_simple_key_t *simple_key = parser->simple_keys.top-1;
1157
1158    if (simple_key->possible)
1159    {
1160        /* If the key is required, it is an error. */
1161
1162        if (simple_key->required) {
1163            return yaml_parser_set_scanner_error(parser,
1164                    "while scanning a simple key", simple_key->mark,
1165                    "could not find expected ':'");
1166        }
1167    }
1168
1169    /* Remove the key from the stack. */
1170
1171    simple_key->possible = 0;
1172
1173    return 1;
1174}
1175
1176/*
1177 * Increase the flow level and resize the simple key list if needed.
1178 */
1179
1180static int
1181yaml_parser_increase_flow_level(yaml_parser_t *parser)
1182{
1183    yaml_simple_key_t empty_simple_key = { 0, 0, 0, { 0, 0, 0 } };
1184
1185    /* Reset the simple key on the next level. */
1186
1187    if (!PUSH(parser, parser->simple_keys, empty_simple_key))
1188        return 0;
1189
1190    /* Increase the flow level. */
1191
1192    parser->flow_level++;
1193
1194    return 1;
1195}
1196
1197/*
1198 * Decrease the flow level.
1199 */
1200
1201static int
1202yaml_parser_decrease_flow_level(yaml_parser_t *parser)
1203{
1204    yaml_simple_key_t dummy_key;    /* Used to eliminate a compiler warning. */
1205
1206    if (parser->flow_level) {
1207        parser->flow_level --;
1208        dummy_key = POP(parser, parser->simple_keys);
1209    }
1210
1211    return 1;
1212}
1213
1214/*
1215 * Push the current indentation level to the stack and set the new level
1216 * the current column is greater than the indentation level.  In this case,
1217 * append or insert the specified token into the token queue.
1218 * 
1219 */
1220
1221static int
1222yaml_parser_roll_indent(yaml_parser_t *parser, int column,
1223        int number, yaml_token_type_t type, yaml_mark_t mark)
1224{
1225    yaml_token_t token;
1226
1227    /* In the flow context, do nothing. */
1228
1229    if (parser->flow_level)
1230        return 1;
1231
1232    if (parser->indent < column)
1233    {
1234        /*
1235         * Push the current indentation level to the stack and set the new
1236         * indentation level.
1237         */
1238
1239        if (!PUSH(parser, parser->indents, parser->indent))
1240            return 0;
1241
1242        parser->indent = column;
1243
1244        /* Create a token and insert it into the queue. */
1245
1246        TOKEN_INIT(token, type, mark, mark);
1247
1248        if (number == -1) {
1249            if (!ENQUEUE(parser, parser->tokens, token))
1250                return 0;
1251        }
1252        else {
1253            if (!QUEUE_INSERT(parser,
1254                        parser->tokens, number - parser->tokens_parsed, token))
1255                return 0;
1256        }
1257    }
1258
1259    return 1;
1260}
1261
1262/*
1263 * Pop indentation levels from the indents stack until the current level
1264 * becomes less or equal to the column.  For each intendation level, append
1265 * the BLOCK-END token.
1266 */
1267
1268
1269static int
1270yaml_parser_unroll_indent(yaml_parser_t *parser, int column)
1271{
1272    yaml_token_t token;
1273
1274    /* In the flow context, do nothing. */
1275
1276    if (parser->flow_level)
1277        return 1;
1278
1279    /* Loop through the intendation levels in the stack. */
1280
1281    while (parser->indent > column)
1282    {
1283        /* Create a token and append it to the queue. */
1284
1285        TOKEN_INIT(token, YAML_BLOCK_END_TOKEN, parser->mark, parser->mark);
1286
1287        if (!ENQUEUE(parser, parser->tokens, token))
1288            return 0;
1289
1290        /* Pop the indentation level. */
1291
1292        parser->indent = POP(parser, parser->indents);
1293    }
1294
1295    return 1;
1296}
1297
1298/*
1299 * Initialize the scanner and produce the STREAM-START token.
1300 */
1301
1302static int
1303yaml_parser_fetch_stream_start(yaml_parser_t *parser)
1304{
1305    yaml_simple_key_t simple_key = { 0, 0, 0, { 0, 0, 0 } };
1306    yaml_token_t token;
1307
1308    /* Set the initial indentation. */
1309
1310    parser->indent = -1;
1311
1312    /* Initialize the simple key stack. */
1313
1314    if (!PUSH(parser, parser->simple_keys, simple_key))
1315        return 0;
1316
1317    /* A simple key is allowed at the beginning of the stream. */
1318
1319    parser->simple_key_allowed = 1;
1320
1321    /* We have started. */
1322
1323    parser->stream_start_produced = 1;
1324
1325    /* Create the STREAM-START token and append it to the queue. */
1326
1327    STREAM_START_TOKEN_INIT(token, parser->encoding,
1328            parser->mark, parser->mark);
1329
1330    if (!ENQUEUE(parser, parser->tokens, token))
1331        return 0;
1332
1333    return 1;
1334}
1335
1336/*
1337 * Produce the STREAM-END token and shut down the scanner.
1338 */
1339
1340static int
1341yaml_parser_fetch_stream_end(yaml_parser_t *parser)
1342{
1343    yaml_token_t token;
1344
1345    /* Force new line. */
1346
1347    if (parser->mark.column != 0) {
1348        parser->mark.column = 0;
1349        parser->mark.line ++;
1350    }
1351
1352    /* Reset the indentation level. */
1353
1354    if (!yaml_parser_unroll_indent(parser, -1))
1355        return 0;
1356
1357    /* Reset simple keys. */
1358
1359    if (!yaml_parser_remove_simple_key(parser))
1360        return 0;
1361
1362    parser->simple_key_allowed = 0;
1363
1364    /* Create the STREAM-END token and append it to the queue. */
1365
1366    STREAM_END_TOKEN_INIT(token, parser->mark, parser->mark);
1367
1368    if (!ENQUEUE(parser, parser->tokens, token))
1369        return 0;
1370
1371    return 1;
1372}
1373
1374/*
1375 * Produce a VERSION-DIRECTIVE or TAG-DIRECTIVE token.
1376 */
1377
1378static int
1379yaml_parser_fetch_directive(yaml_parser_t *parser)
1380{
1381    yaml_token_t token;
1382
1383    /* Reset the indentation level. */
1384
1385    if (!yaml_parser_unroll_indent(parser, -1))
1386        return 0;
1387
1388    /* Reset simple keys. */
1389
1390    if (!yaml_parser_remove_simple_key(parser))
1391        return 0;
1392
1393    parser->simple_key_allowed = 0;
1394
1395    /* Create the YAML-DIRECTIVE or TAG-DIRECTIVE token. */
1396
1397    if (!yaml_parser_scan_directive(parser, &token))
1398        return 0;
1399
1400    /* Append the token to the queue. */
1401
1402    if (!ENQUEUE(parser, parser->tokens, token)) {
1403        yaml_token_delete(&token);
1404        return 0;
1405    }
1406
1407    return 1;
1408}
1409
1410/*
1411 * Produce the DOCUMENT-START or DOCUMENT-END token.
1412 */
1413
1414static int
1415yaml_parser_fetch_document_indicator(yaml_parser_t *parser,
1416        yaml_token_type_t type)
1417{
1418    yaml_mark_t start_mark, end_mark;
1419    yaml_token_t token;
1420
1421    /* Reset the indentation level. */
1422
1423    if (!yaml_parser_unroll_indent(parser, -1))
1424        return 0;
1425
1426    /* Reset simple keys. */
1427
1428    if (!yaml_parser_remove_simple_key(parser))
1429        return 0;
1430
1431    parser->simple_key_allowed = 0;
1432
1433    /* Consume the token. */
1434
1435    start_mark = parser->mark;
1436
1437    SKIP(parser);
1438    SKIP(parser);
1439    SKIP(parser);
1440
1441    end_mark = parser->mark;
1442
1443    /* Create the DOCUMENT-START or DOCUMENT-END token. */
1444
1445    TOKEN_INIT(token, type, start_mark, end_mark);
1446
1447    /* Append the token to the queue. */
1448
1449    if (!ENQUEUE(parser, parser->tokens, token))
1450        return 0;
1451
1452    return 1;
1453}
1454
1455/*
1456 * Produce the FLOW-SEQUENCE-START or FLOW-MAPPING-START token.
1457 */
1458
1459static int
1460yaml_parser_fetch_flow_collection_start(yaml_parser_t *parser,
1461        yaml_token_type_t type)
1462{
1463    yaml_mark_t start_mark, end_mark;
1464    yaml_token_t token;
1465
1466    /* The indicators '[' and '{' may start a simple key. */
1467
1468    if (!yaml_parser_save_simple_key(parser))
1469        return 0;
1470
1471    /* Increase the flow level. */
1472
1473    if (!yaml_parser_increase_flow_level(parser))
1474        return 0;
1475
1476    /* A simple key may follow the indicators '[' and '{'. */
1477
1478    parser->simple_key_allowed = 1;
1479
1480    /* Consume the token. */
1481
1482    start_mark = parser->mark;
1483    SKIP(parser);
1484    end_mark = parser->mark;
1485
1486    /* Create the FLOW-SEQUENCE-START of FLOW-MAPPING-START token. */
1487
1488    TOKEN_INIT(token, type, start_mark, end_mark);
1489
1490    /* Append the token to the queue. */
1491
1492    if (!ENQUEUE(parser, parser->tokens, token))
1493        return 0;
1494
1495    return 1;
1496}
1497
1498/*
1499 * Produce the FLOW-SEQUENCE-END or FLOW-MAPPING-END token.
1500 */
1501
1502static int
1503yaml_parser_fetch_flow_collection_end(yaml_parser_t *parser,
1504        yaml_token_type_t type)
1505{
1506    yaml_mark_t start_mark, end_mark;
1507    yaml_token_t token;
1508
1509    /* Reset any potential simple key on the current flow level. */
1510
1511    if (!yaml_parser_remove_simple_key(parser))
1512        return 0;
1513
1514    /* Decrease the flow level. */
1515
1516    if (!yaml_parser_decrease_flow_level(parser))
1517        return 0;
1518
1519    /* No simple keys after the indicators ']' and '}'. */
1520
1521    parser->simple_key_allowed = 0;
1522
1523    /* Consume the token. */
1524
1525    start_mark = parser->mark;
1526    SKIP(parser);
1527    end_mark = parser->mark;
1528
1529    /* Create the FLOW-SEQUENCE-END of FLOW-MAPPING-END token. */
1530
1531    TOKEN_INIT(token, type, start_mark, end_mark);
1532
1533    /* Append the token to the queue. */
1534
1535    if (!ENQUEUE(parser, parser->tokens, token))
1536        return 0;
1537
1538    return 1;
1539}
1540
1541/*
1542 * Produce the FLOW-ENTRY token.
1543 */
1544
1545static int
1546yaml_parser_fetch_flow_entry(yaml_parser_t *parser)
1547{
1548    yaml_mark_t start_mark, end_mark;
1549    yaml_token_t token;
1550
1551    /* Reset any potential simple keys on the current flow level. */
1552
1553    if (!yaml_parser_remove_simple_key(parser))
1554        return 0;
1555
1556    /* Simple keys are allowed after ','. */
1557
1558    parser->simple_key_allowed = 1;
1559
1560    /* Consume the token. */
1561
1562    start_mark = parser->mark;
1563    SKIP(parser);
1564    end_mark = parser->mark;
1565
1566    /* Create the FLOW-ENTRY token and append it to the queue. */
1567
1568    TOKEN_INIT(token, YAML_FLOW_ENTRY_TOKEN, start_mark, end_mark);
1569
1570    if (!ENQUEUE(parser, parser->tokens, token))
1571        return 0;
1572
1573    return 1;
1574}
1575
1576/*
1577 * Produce the BLOCK-ENTRY token.
1578 */
1579
1580static int
1581yaml_parser_fetch_block_entry(yaml_parser_t *parser)
1582{
1583    yaml_mark_t start_mark, end_mark;
1584    yaml_token_t token;
1585
1586    /* Check if the scanner is in the block context. */
1587
1588    if (!parser->flow_level)
1589    {
1590        /* Check if we are allowed to start a new entry. */
1591
1592        if (!parser->simple_key_allowed) {
1593            return yaml_parser_set_scanner_error(parser, NULL, parser->mark,
1594                    "block sequence entries are not allowed in this context");
1595        }
1596
1597        /* Add the BLOCK-SEQUENCE-START token if needed. */
1598
1599        if (!yaml_parser_roll_indent(parser, parser->mark.column, -1,
1600                    YAML_BLOCK_SEQUENCE_START_TOKEN, parser->mark))
1601            return 0;
1602    }
1603    else
1604    {
1605        /*
1606         * It is an error for the '-' indicator to occur in the flow context,
1607         * but we let the Parser detect and report about it because the Parser
1608         * is able to point to the context.
1609         */
1610    }
1611
1612    /* Reset any potential simple keys on the current flow level. */
1613
1614    if (!yaml_parser_remove_simple_key(parser))
1615        return 0;
1616
1617    /* Simple keys are allowed after '-'. */
1618
1619    parser->simple_key_allowed = 1;
1620
1621    /* Consume the token. */
1622
1623    start_mark = parser->mark;
1624    SKIP(parser);
1625    end_mark = parser->mark;
1626
1627    /* Create the BLOCK-ENTRY token and append it to the queue. */
1628
1629    TOKEN_INIT(token, YAML_BLOCK_ENTRY_TOKEN, start_mark, end_mark);
1630
1631    if (!ENQUEUE(parser, parser->tokens, token))
1632        return 0;
1633
1634    return 1;
1635}
1636
1637/*
1638 * Produce the KEY token.
1639 */
1640
1641static int
1642yaml_parser_fetch_key(yaml_parser_t *parser)
1643{
1644    yaml_mark_t start_mark, end_mark;
1645    yaml_token_t token;
1646
1647    /* In the block context, additional checks are required. */
1648
1649    if (!parser->flow_level)
1650    {
1651        /* Check if we are allowed to start a new key (not nessesary simple). */
1652
1653        if (!parser->simple_key_allowed) {
1654            return yaml_parser_set_scanner_error(parser, NULL, parser->mark,
1655                    "mapping keys are not allowed in this context");
1656        }
1657
1658        /* Add the BLOCK-MAPPING-START token if needed. */
1659
1660        if (!yaml_parser_roll_indent(parser, parser->mark.column, -1,
1661                    YAML_BLOCK_MAPPING_START_TOKEN, parser->mark))
1662            return 0;
1663    }
1664
1665    /* Reset any potential simple keys on the current flow level. */
1666
1667    if (!yaml_parser_remove_simple_key(parser))
1668        return 0;
1669
1670    /* Simple keys are allowed after '?' in the block context. */
1671
1672    parser->simple_key_allowed = (!parser->flow_level);
1673
1674    /* Consume the token. */
1675
1676    start_mark = parser->mark;
1677    SKIP(parser);
1678    end_mark = parser->mark;
1679
1680    /* Create the KEY token and append it to the queue. */
1681
1682    TOKEN_INIT(token, YAML_KEY_TOKEN, start_mark, end_mark);
1683
1684    if (!ENQUEUE(parser, parser->tokens, token))
1685        return 0;
1686
1687    return 1;
1688}
1689
1690/*
1691 * Produce the VALUE token.
1692 */
1693
1694static int
1695yaml_parser_fetch_value(yaml_parser_t *parser)
1696{
1697    yaml_mark_t start_mark, end_mark;
1698    yaml_token_t token;
1699    yaml_simple_key_t *simple_key = parser->simple_keys.top-1;
1700
1701    /* Have we found a simple key? */
1702
1703    if (simple_key->possible)
1704    {
1705
1706        /* Create the KEY token and insert it into the queue. */
1707
1708        TOKEN_INIT(token, YAML_KEY_TOKEN, simple_key->mark, simple_key->mark);
1709
1710        if (!QUEUE_INSERT(parser, parser->tokens,
1711                    simple_key->token_number - parser->tokens_parsed, token))
1712            return 0;
1713
1714        /* In the block context, we may need to add the BLOCK-MAPPING-START token. */
1715
1716        if (!yaml_parser_roll_indent(parser, simple_key->mark.column,
1717                    simple_key->token_number,
1718                    YAML_BLOCK_MAPPING_START_TOKEN, simple_key->mark))
1719            return 0;
1720
1721        /* Remove the simple key. */
1722
1723        simple_key->possible = 0;
1724
1725        /* A simple key cannot follow another simple key. */
1726
1727        parser->simple_key_allowed = 0;
1728    }
1729    else
1730    {
1731        /* The ':' indicator follows a complex key. */
1732
1733        /* In the block context, extra checks are required. */
1734
1735        if (!parser->flow_level)
1736        {
1737            /* Check if we are allowed to start a complex value. */
1738
1739            if (!parser->simple_key_allowed) {
1740                return yaml_parser_set_scanner_error(parser, NULL, parser->mark,
1741                        "mapping values are not allowed in this context");
1742            }
1743
1744            /* Add the BLOCK-MAPPING-START token if needed. */
1745
1746            if (!yaml_parser_roll_indent(parser, parser->mark.column, -1,
1747                        YAML_BLOCK_MAPPING_START_TOKEN, parser->mark))
1748                return 0;
1749        }
1750
1751        /* Simple keys after ':' are allowed in the block context. */
1752
1753        parser->simple_key_allowed = (!parser->flow_level);
1754    }
1755
1756    /* Consume the token. */
1757
1758    start_mark = parser->mark;
1759    SKIP(parser);
1760    end_mark = parser->mark;
1761
1762    /* Create the VALUE token and append it to the queue. */
1763
1764    TOKEN_INIT(token, YAML_VALUE_TOKEN, start_mark, end_mark);
1765
1766    if (!ENQUEUE(parser, parser->tokens, token))
1767        return 0;
1768
1769    return 1;
1770}
1771
1772/*
1773 * Produce the ALIAS or ANCHOR token.
1774 */
1775
1776static int
1777yaml_parser_fetch_anchor(yaml_parser_t *parser, yaml_token_type_t type)
1778{
1779    yaml_token_t token;
1780
1781    /* An anchor or an alias could be a simple key. */
1782
1783    if (!yaml_parser_save_simple_key(parser))
1784        return 0;
1785
1786    /* A simple key cannot follow an anchor or an alias. */
1787
1788    parser->simple_key_allowed = 0;
1789
1790    /* Cre…

Large files files are truncated, but you can click here to view the full file