/*
 * This file is part of Hubbub.
 * Licensed under the MIT License,
 *                http://www.opensource.org/licenses/mit-license.php
 * Copyright 2007 John-Mark Bell <jmb@netsurf-browser.org>
 * Copyright 2008 Andrew Sidwell <takkaria@netsurf-browser.org>
 */
#include <assert.h>
#include <stdbool.h>
#include <string.h>

#include <stdio.h>

#include <parserutils/charset/utf8.h>

#include "utils/parserutilserror.h"
#include "utils/utils.h"

#include "tokeniser/entities.h"
#include "tokeniser/tokeniser.h"

/**
 * Table of mappings between Windows-1252 codepoints 128-159 and UCS4
 */
static const uint32_t cp1252Table[32] = {
	0x20AC, 0xFFFD, 0x201A, 0x0192, 0x201E, 0x2026, 0x2020, 0x2021,
	0x02C6, 0x2030, 0x0160, 0x2039, 0x0152, 0xFFFD, 0x017D, 0xFFFD,
	0xFFFD, 0x2018, 0x2019, 0x201C, 0x201D, 0x2022, 0x2013, 0x2014,
	0x02DC, 0x2122, 0x0161, 0x203A, 0x0153, 0xFFFD, 0x017E, 0x0178
};

/**
 * UTF-8 encoding of U+FFFD REPLACEMENT CHARACTER
 */
static const uint8_t u_fffd[3] = { '\xEF', '\xBF', '\xBD' };
static const hubbub_string u_fffd_str = { u_fffd, sizeof(u_fffd) };


/**
 * String for when we want to emit newlines
 */
static const uint8_t lf = '\n';
static const hubbub_string lf_str = { &lf, 1 };


/**
 * Tokeniser states
 */
typedef enum hubbub_tokeniser_state {
	STATE_DATA,
	STATE_CHARACTER_REFERENCE_DATA,
	STATE_TAG_OPEN,
	STATE_CLOSE_TAG_OPEN,
	STATE_TAG_NAME,
	STATE_BEFORE_ATTRIBUTE_NAME,
	STATE_ATTRIBUTE_NAME,
	STATE_AFTER_ATTRIBUTE_NAME,
	STATE_BEFORE_ATTRIBUTE_VALUE,
	STATE_ATTRIBUTE_VALUE_DQ,
	STATE_ATTRIBUTE_VALUE_SQ,
	STATE_ATTRIBUTE_VALUE_UQ,
	STATE_CHARACTER_REFERENCE_IN_ATTRIBUTE_VALUE,
	STATE_AFTER_ATTRIBUTE_VALUE_Q,
	STATE_SELF_CLOSING_START_TAG,
	STATE_BOGUS_COMMENT,
	STATE_MARKUP_DECLARATION_OPEN,
	STATE_MATCH_COMMENT,
	STATE_COMMENT_START,
	STATE_COMMENT_START_DASH,
	STATE_COMMENT,
	STATE_COMMENT_END_DASH,
	STATE_COMMENT_END,
	STATE_MATCH_DOCTYPE,
	STATE_DOCTYPE,
	STATE_BEFORE_DOCTYPE_NAME,
	STATE_DOCTYPE_NAME,
	STATE_AFTER_DOCTYPE_NAME,
	STATE_MATCH_PUBLIC,
	STATE_BEFORE_DOCTYPE_PUBLIC,
	STATE_DOCTYPE_PUBLIC_DQ,
	STATE_DOCTYPE_PUBLIC_SQ,
	STATE_AFTER_DOCTYPE_PUBLIC,
	STATE_MATCH_SYSTEM,
	STATE_BEFORE_DOCTYPE_SYSTEM,
	STATE_DOCTYPE_SYSTEM_DQ,
	STATE_DOCTYPE_SYSTEM_SQ,
	STATE_AFTER_DOCTYPE_SYSTEM,
	STATE_BOGUS_DOCTYPE,
	STATE_MATCH_CDATA,
	STATE_CDATA_BLOCK,
	STATE_NUMBERED_ENTITY,
	STATE_NAMED_ENTITY
} hubbub_tokeniser_state;

/**
 * Context for tokeniser
 */
typedef struct hubbub_tokeniser_context {
	size_t pending;				/**< Count of pending chars */

	hubbub_string current_comment;		/**< Current comment text */

	hubbub_token_type current_tag_type;	/**< Type of current_tag */
	hubbub_tag current_tag;			/**< Current tag */
	hubbub_doctype current_doctype;		/**< Current doctype */
	hubbub_tokeniser_state prev_state;	/**< Previous state */

	uint8_t last_start_tag_name[10];	/**< Name of the last start tag
						 * emitted */
	size_t last_start_tag_len;		/**< Length of last start tag */

	struct {
		uint32_t count;
		bool match;
	} close_tag_match;			/**< State for matching close 
						 * tags */

	struct {
		uint32_t count;			/**< Index into "DOCTYPE" */
	} match_doctype;			/**< State for matching doctype */

	struct {
		uint32_t count;			/**< Index into "[CDATA[" */
		uint32_t end;			/**< Index into "]]>" */
	} match_cdata;				/**< State for matching cdata */

	struct {
		size_t offset;			/**< Offset in buffer */
		uint32_t length;		/**< Length of entity */
		uint32_t codepoint;		/**< UCS4 codepoint */
		bool complete;			/**< True if match complete */

		uint32_t poss_length;		/**< Optimistic length
						 * when matching named
						 * character references */
		uint8_t base;			/**< Base for numeric
						 * entities */
		void *context;			/**< Context for named
						 * entity search */
		size_t prev_len;		/**< Previous byte length
						 * of str */
		bool had_data;			/**< Whether we read
						 * anything after &#(x)? */
		bool overflow;			/**< Whether this entity has
						 * has overflowed the maximum
						 * numeric entity value */
		hubbub_tokeniser_state return_state;	/**< State we were
							 * called from */
	} match_entity;				/**< Entity matching state */

	struct {
		uint32_t line;			/**< Current line of input */
		uint32_t col;			/**< Current character in
						 * line */
	} position;				/**< Position in source data */

	uint32_t allowed_char;			/**< Used for quote matching */
} hubbub_tokeniser_context;

/**
 * Tokeniser data structure
 */
struct hubbub_tokeniser {
	hubbub_tokeniser_state state;	/**< Current tokeniser state */
	hubbub_content_model content_model;	/**< Current content
						 * model flag */
	bool escape_flag;		/**< Escape flag **/
	bool process_cdata_section;	/**< Whether to process CDATA sections*/

	parserutils_inputstream *input;	/**< Input stream */
	parserutils_buffer *buffer;	/**< Input buffer */

	hubbub_tokeniser_context context;	/**< Tokeniser context */

	hubbub_token_handler token_handler;	/**< Token handling callback */
	void *token_pw;				/**< Token handler data */

	hubbub_error_handler error_handler;	/**< Error handling callback */
	void *error_pw;				/**< Error handler data */

	hubbub_allocator_fn alloc;	/**< Memory (de)allocation function */
	void *alloc_pw;			/**< Client private data */
};

static hubbub_error hubbub_tokeniser_handle_data(hubbub_tokeniser *tokeniser);
static hubbub_error hubbub_tokeniser_handle_character_reference_data(
		hubbub_tokeniser *tokeniser);
static hubbub_error hubbub_tokeniser_handle_tag_open(
		hubbub_tokeniser *tokeniser);
static hubbub_error hubbub_tokeniser_handle_close_tag_open(
		hubbub_tokeniser *tokeniser);
static hubbub_error hubbub_tokeniser_handle_tag_name(
		hubbub_tokeniser *tokeniser);
static hubbub_error hubbub_tokeniser_handle_before_attribute_name(
		hubbub_tokeniser *tokeniser);
static hubbub_error hubbub_tokeniser_handle_attribute_name(
		hubbub_tokeniser *tokeniser);
static hubbub_error hubbub_tokeniser_handle_after_attribute_name(
		hubbub_tokeniser *tokeniser);
static hubbub_error hubbub_tokeniser_handle_before_attribute_value(
		hubbub_tokeniser *tokeniser);
static hubbub_error hubbub_tokeniser_handle_attribute_value_dq(
		hubbub_tokeniser *tokeniser);
static hubbub_error hubbub_tokeniser_handle_attribute_value_sq(
		hubbub_tokeniser *tokeniser);
static hubbub_error hubbub_tokeniser_handle_attribute_value_uq(
		hubbub_tokeniser *tokeniser);
static hubbub_error hubbub_tokeniser_handle_character_reference_in_attribute_value(
		hubbub_tokeniser *tokeniser);
static hubbub_error hubbub_tokeniser_handle_after_attribute_value_q(
		hubbub_tokeniser *tokeniser);
static hubbub_error hubbub_tokeniser_handle_self_closing_start_tag(
		hubbub_tokeniser *tokeniser);
static hubbub_error hubbub_tokeniser_handle_bogus_comment(
		hubbub_tokeniser *tokeniser);
static hubbub_error hubbub_tokeniser_handle_markup_declaration_open(
		hubbub_tokeniser *tokeniser);
static hubbub_error hubbub_tokeniser_handle_match_comment(
		hubbub_tokeniser *tokeniser);
static hubbub_error hubbub_tokeniser_handle_comment(
		hubbub_tokeniser *tokeniser);
static hubbub_error hubbub_tokeniser_handle_match_doctype(
		hubbub_tokeniser *tokeniser);
static hubbub_error hubbub_tokeniser_handle_doctype(
		hubbub_tokeniser *tokeniser);
static hubbub_error hubbub_tokeniser_handle_before_doctype_name(
		hubbub_tokeniser *tokeniser);
static hubbub_error hubbub_tokeniser_handle_doctype_name(
		hubbub_tokeniser *tokeniser);
static hubbub_error hubbub_tokeniser_handle_after_doctype_name(
		hubbub_tokeniser *tokeniser);
static hubbub_error hubbub_tokeniser_handle_match_public(
		hubbub_tokeniser *tokeniser);
static hubbub_error hubbub_tokeniser_handle_before_doctype_public(
		hubbub_tokeniser *tokeniser);
static hubbub_error hubbub_tokeniser_handle_doctype_public_dq(
		hubbub_tokeniser *tokeniser);
static hubbub_error hubbub_tokeniser_handle_doctype_public_sq(
		hubbub_tokeniser *tokeniser);
static hubbub_error hubbub_tokeniser_handle_after_doctype_public(
		hubbub_tokeniser *tokeniser);
static hubbub_error hubbub_tokeniser_handle_match_system(
		hubbub_tokeniser *tokeniser);
static hubbub_error hubbub_tokeniser_handle_before_doctype_system(
		hubbub_tokeniser *tokeniser);
static hubbub_error hubbub_tokeniser_handle_doctype_system_dq(
		hubbub_tokeniser *tokeniser);
static hubbub_error hubbub_tokeniser_handle_doctype_system_sq(
		hubbub_tokeniser *tokeniser);
static hubbub_error hubbub_tokeniser_handle_after_doctype_system(
		hubbub_tokeniser *tokeniser);
static hubbub_error hubbub_tokeniser_handle_bogus_doctype(
		hubbub_tokeniser *tokeniser);
static hubbub_error hubbub_tokeniser_handle_match_cdata(
		hubbub_tokeniser *tokeniser);
static hubbub_error hubbub_tokeniser_handle_cdata_block(
		hubbub_tokeniser *tokeniser);
static hubbub_error hubbub_tokeniser_consume_character_reference(
		hubbub_tokeniser *tokeniser, size_t off);
static hubbub_error hubbub_tokeniser_handle_numbered_entity(
		hubbub_tokeniser *tokeniser);
static hubbub_error hubbub_tokeniser_handle_named_entity(
		hubbub_tokeniser *tokeniser);

static inline hubbub_error emit_character_token(hubbub_tokeniser *tokeniser,
		const hubbub_string *chars);
static inline hubbub_error emit_current_chars(hubbub_tokeniser *tokeniser);
static inline hubbub_error emit_current_tag(hubbub_tokeniser *tokeniser);
static inline hubbub_error emit_current_comment(hubbub_tokeniser *tokeniser);
static inline hubbub_error emit_current_doctype(hubbub_tokeniser *tokeniser,
		bool force_quirks);
static hubbub_error hubbub_tokeniser_emit_token(hubbub_tokeniser *tokeniser,
		hubbub_token *token);

/**
 * Create a hubbub tokeniser
 *
 * \param input      Input stream instance
 * \param alloc      Memory (de)allocation function
 * \param pw         Pointer to client-specific private data (may be NULL)
 * \param tokeniser  Pointer to location to receive tokeniser instance
 * \return HUBBUB_OK on success,
 *         HUBBUB_BADPARM on bad parameters,
 *         HUBBUB_NOMEM on memory exhaustion
 */
hubbub_error hubbub_tokeniser_create(parserutils_inputstream *input,
		hubbub_allocator_fn alloc, void *pw, 
		hubbub_tokeniser **tokeniser)
{
	parserutils_error perror;
	hubbub_tokeniser *tok;

	if (input == NULL || alloc == NULL || tokeniser == NULL)
		return HUBBUB_BADPARM;

	tok = alloc(NULL, sizeof(hubbub_tokeniser), pw);
	if (tok == NULL)
		return HUBBUB_NOMEM;

	perror = parserutils_buffer_create(alloc, pw, &tok->buffer);
	if (perror != PARSERUTILS_OK) {
		alloc(tok, 0, pw);
		return hubbub_error_from_parserutils_error(perror);
	}

	tok->state = STATE_DATA;
	tok->content_model = HUBBUB_CONTENT_MODEL_PCDATA;

	tok->escape_flag = false;
	tok->process_cdata_section = false;

	tok->input = input;

	tok->token_handler = NULL;
	tok->token_pw = NULL;

	tok->error_handler = NULL;
	tok->error_pw = NULL;

	tok->alloc = alloc;
	tok->alloc_pw = pw;

	memset(&tok->context, 0, sizeof(hubbub_tokeniser_context));

	*tokeniser = tok;

	return HUBBUB_OK;
}

/**
 * Destroy a hubbub tokeniser
 *
 * \param tokeniser  The tokeniser instance to destroy
 * \return HUBBUB_OK on success, appropriate error otherwise
 */
hubbub_error hubbub_tokeniser_destroy(hubbub_tokeniser *tokeniser)
{
	if (tokeniser == NULL)
		return HUBBUB_BADPARM;

	if (tokeniser->context.current_tag.attributes != NULL) {
		tokeniser->alloc(tokeniser->context.current_tag.attributes,
				0, tokeniser->alloc_pw);
	}

	parserutils_buffer_destroy(tokeniser->buffer);

	tokeniser->alloc(tokeniser, 0, tokeniser->alloc_pw);

	return HUBBUB_OK;
}

/**
 * Configure a hubbub tokeniser
 *
 * \param tokeniser  The tokeniser instance to configure
 * \param type       The option type to set
 * \param params     Option-specific parameters
 * \return HUBBUB_OK on success, appropriate error otherwise
 */
hubbub_error hubbub_tokeniser_setopt(hubbub_tokeniser *tokeniser,
		hubbub_tokeniser_opttype type,
		hubbub_tokeniser_optparams *params)
{
	if (tokeniser == NULL || params == NULL)
		return HUBBUB_BADPARM;

	switch (type) {
	case HUBBUB_TOKENISER_TOKEN_HANDLER:
		tokeniser->token_handler = params->token_handler.handler;
		tokeniser->token_pw = params->token_handler.pw;
		break;
	case HUBBUB_TOKENISER_ERROR_HANDLER:
		tokeniser->error_handler = params->error_handler.handler;
		tokeniser->error_pw = params->error_handler.pw;
		break;
	case HUBBUB_TOKENISER_CONTENT_MODEL:
		tokeniser->content_model = params->content_model.model;
		break;
	case HUBBUB_TOKENISER_PROCESS_CDATA:
		tokeniser->process_cdata_section = params->process_cdata;
		break;
	}

	return HUBBUB_OK;
}

/**
 * Process remaining data in the input stream
 *
 * \param tokeniser  The tokeniser instance to invoke
 * \return HUBBUB_OK on success, appropriate error otherwise
 */
hubbub_error hubbub_tokeniser_run(hubbub_tokeniser *tokeniser)
{
	hubbub_error cont = HUBBUB_OK;

	if (tokeniser == NULL)
		return HUBBUB_BADPARM;

#if 0
#define state(x) \
		case x: \
			printf( #x "\n");
#else
#define state(x) \
		case x:
#endif

	while (cont == HUBBUB_OK) {
		switch (tokeniser->state) {
		state(STATE_DATA)
			cont = hubbub_tokeniser_handle_data(tokeniser);
			break;
		state(STATE_CHARACTER_REFERENCE_DATA)
			cont = hubbub_tokeniser_handle_character_reference_data(
					tokeniser);
			break;
		state(STATE_TAG_OPEN)
			cont = hubbub_tokeniser_handle_tag_open(tokeniser);
			break;
		state(STATE_CLOSE_TAG_OPEN)
			cont = hubbub_tokeniser_handle_close_tag_open(
					tokeniser);
			break;
		state(STATE_TAG_NAME)
			cont = hubbub_tokeniser_handle_tag_name(tokeniser);
			break;
		state(STATE_BEFORE_ATTRIBUTE_NAME)
			cont = hubbub_tokeniser_handle_before_attribute_name(
					tokeniser);
			break;
		state(STATE_ATTRIBUTE_NAME)
			cont = hubbub_tokeniser_handle_attribute_name(
					tokeniser);
			break;
		state(STATE_AFTER_ATTRIBUTE_NAME)
			cont = hubbub_tokeniser_handle_after_attribute_name(
					tokeniser);
			break;
		state(STATE_BEFORE_ATTRIBUTE_VALUE)
			cont = hubbub_tokeniser_handle_before_attribute_value(
					tokeniser);
			break;
		state(STATE_ATTRIBUTE_VALUE_DQ)
			cont = hubbub_tokeniser_handle_attribute_value_dq(
					tokeniser);
			break;
		state(STATE_ATTRIBUTE_VALUE_SQ)
			cont = hubbub_tokeniser_handle_attribute_value_sq(
					tokeniser);
			break;
		state(STATE_ATTRIBUTE_VALUE_UQ)
			cont = hubbub_tokeniser_handle_attribute_value_uq(
					tokeniser);
			break;
		state(STATE_CHARACTER_REFERENCE_IN_ATTRIBUTE_VALUE)
			cont = hubbub_tokeniser_handle_character_reference_in_attribute_value(
					tokeniser);
			break;
		state(STATE_AFTER_ATTRIBUTE_VALUE_Q)
			cont = hubbub_tokeniser_handle_after_attribute_value_q(
					tokeniser);
			break;
		state(STATE_SELF_CLOSING_START_TAG)
			cont = hubbub_tokeniser_handle_self_closing_start_tag(
					tokeniser);
			break;
		state(STATE_BOGUS_COMMENT)
			cont = hubbub_tokeniser_handle_bogus_comment(
					tokeniser);
			break;
		state(STATE_MARKUP_DECLARATION_OPEN)
			cont = hubbub_tokeniser_handle_markup_declaration_open(
					tokeniser);
			break;
		state(STATE_MATCH_COMMENT)
			cont = hubbub_tokeniser_handle_match_comment(
					tokeniser);
			break;
		case STATE_COMMENT_START:
		case STATE_COMMENT_START_DASH:
		case STATE_COMMENT:
		case STATE_COMMENT_END_DASH:
		case STATE_COMMENT_END:
			cont = hubbub_tokeniser_handle_comment(tokeniser);
			break;
		state(STATE_MATCH_DOCTYPE)
			cont = hubbub_tokeniser_handle_match_doctype(
					tokeniser);
			break;
		state(STATE_DOCTYPE)
			cont = hubbub_tokeniser_handle_doctype(tokeniser);
			break;
		state(STATE_BEFORE_DOCTYPE_NAME)
			cont = hubbub_tokeniser_handle_before_doctype_name(
					tokeniser);
			break;
		state(STATE_DOCTYPE_NAME)
			cont = hubbub_tokeniser_handle_doctype_name(
					tokeniser);
			break;
		state(STATE_AFTER_DOCTYPE_NAME)
			cont = hubbub_tokeniser_handle_after_doctype_name(
					tokeniser);
			break;

		state(STATE_MATCH_PUBLIC)
			cont = hubbub_tokeniser_handle_match_public(
					tokeniser);
			break;
		state(STATE_BEFORE_DOCTYPE_PUBLIC)
			cont = hubbub_tokeniser_handle_before_doctype_public(
					tokeniser);
			break;
		state(STATE_DOCTYPE_PUBLIC_DQ)
			cont = hubbub_tokeniser_handle_doctype_public_dq(
					tokeniser);
			break;
		state(STATE_DOCTYPE_PUBLIC_SQ)
			cont = hubbub_tokeniser_handle_doctype_public_sq(
					tokeniser);
			break;
		state(STATE_AFTER_DOCTYPE_PUBLIC)
			cont = hubbub_tokeniser_handle_after_doctype_public(
					tokeniser);
			break;
		state(STATE_MATCH_SYSTEM)
			cont = hubbub_tokeniser_handle_match_system(
					tokeniser);
			break;
		state(STATE_BEFORE_DOCTYPE_SYSTEM)
			cont = hubbub_tokeniser_handle_before_doctype_system(
					tokeniser);
			break;
		state(STATE_DOCTYPE_SYSTEM_DQ)
			cont = hubbub_tokeniser_handle_doctype_system_dq(
					tokeniser);
			break;
		state(STATE_DOCTYPE_SYSTEM_SQ)
			cont = hubbub_tokeniser_handle_doctype_system_sq(
					tokeniser);
			break;
		state(STATE_AFTER_DOCTYPE_SYSTEM)
			cont = hubbub_tokeniser_handle_after_doctype_system(
					tokeniser);
			break;
		state(STATE_BOGUS_DOCTYPE)
			cont = hubbub_tokeniser_handle_bogus_doctype(
					tokeniser);
			break;
		state(STATE_MATCH_CDATA)
			cont = hubbub_tokeniser_handle_match_cdata(
					tokeniser);
			break;
		state(STATE_CDATA_BLOCK)
			cont = hubbub_tokeniser_handle_cdata_block(
					tokeniser);
			break;
		state(STATE_NUMBERED_ENTITY)
			cont = hubbub_tokeniser_handle_numbered_entity(
					tokeniser);
			break;
		state(STATE_NAMED_ENTITY)
			cont = hubbub_tokeniser_handle_named_entity(
					tokeniser);
			break;
		}
	}

	return (cont == HUBBUB_NEEDDATA) ? HUBBUB_OK : cont;
}


/**
 * Various macros for manipulating buffers.
 *
 * \todo make some of these inline functions (type-safety)
 * \todo document them properly here
 */

#define START_BUF(str, cptr, length) \
	do { \
		parserutils_error perror; \
		perror = parserutils_buffer_append(tokeniser->buffer, \
				(uint8_t *) (cptr), (length)); \
		if (perror != PARSERUTILS_OK) \
			return hubbub_error_from_parserutils_error(perror); \
		(str).len = (length); \
	} while (0)

#define COLLECT(str, cptr, length) \
	do { \
		parserutils_error perror; \
		assert(str.len != 0); \
		perror = parserutils_buffer_append(tokeniser->buffer, \
				(uint8_t *) (cptr), (length)); \
		if (perror != PARSERUTILS_OK) \
			return hubbub_error_from_parserutils_error(perror); \
		(str).len += (length); \
	} while (0)

#define COLLECT_MS(str, cptr, length) \
	do { \
		parserutils_error perror; \
		perror = parserutils_buffer_append(tokeniser->buffer, \
				(uint8_t *) (cptr), (length)); \
		if (perror != PARSERUTILS_OK) \
			return hubbub_error_from_parserutils_error(perror); \
		(str).len += (length); \
	} while (0)


/* this should always be called with an empty "chars" buffer */
hubbub_error hubbub_tokeniser_handle_data(hubbub_tokeniser *tokeniser)
{
	parserutils_error error;
	hubbub_token token;
	const uint8_t *cptr;
	size_t len;

	while ((error = parserutils_inputstream_peek(tokeniser->input,
			tokeniser->context.pending, &cptr, &len)) ==
					PARSERUTILS_OK) {
		const uint8_t c = *cptr;

		if (c == '&' &&
				(tokeniser->content_model == HUBBUB_CONTENT_MODEL_PCDATA ||
				tokeniser->content_model == HUBBUB_CONTENT_MODEL_RCDATA) &&
				tokeniser->escape_flag == false) {
			tokeniser->state =
					STATE_CHARACTER_REFERENCE_DATA;
			/* Don't eat the '&'; it'll be handled by entity
			 * consumption */
			break;
		} else if (c == '-' &&
				tokeniser->escape_flag == false &&
				(tokeniser->content_model ==
						HUBBUB_CONTENT_MODEL_RCDATA ||
				tokeniser->content_model ==
						HUBBUB_CONTENT_MODEL_CDATA) &&
				tokeniser->context.pending >= 3) {
			size_t ignore;
			error = parserutils_inputstream_peek(
					tokeniser->input,
					tokeniser->context.pending - 3,
					&cptr,
					&ignore);

			assert(error == PARSERUTILS_OK);

			if (strncmp((char *)cptr,
					"<!--", SLEN("<!--")) == 0) {
				tokeniser->escape_flag = true;
			}

			tokeniser->context.pending += len;
		} else if (c == '<' && (tokeniser->content_model ==
						HUBBUB_CONTENT_MODEL_PCDATA ||
					((tokeniser->content_model ==
						HUBBUB_CONTENT_MODEL_RCDATA ||
					tokeniser->content_model ==
						HUBBUB_CONTENT_MODEL_CDATA) &&
				tokeniser->escape_flag == false))) {
			if (tokeniser->context.pending > 0) {
				/* Emit any pending characters */
				emit_current_chars(tokeniser);
			}

			/* Buffer '<' */
			tokeniser->context.pending = len;
			tokeniser->state = STATE_TAG_OPEN;
			break;
		} else if (c == '>' && tokeniser->escape_flag == true &&
				(tokeniser->content_model ==
						HUBBUB_CONTENT_MODEL_RCDATA ||
				tokeniser->content_model ==
						HUBBUB_CONTENT_MODEL_CDATA)) {
			/* no need to check that there are enough characters,
			 * since you can only run into this if the flag is
			 * true in the first place, which requires four
			 * characters. */
			error = parserutils_inputstream_peek(
					tokeniser->input,
					tokeniser->context.pending - 2,
					&cptr,
					&len);

			assert(error == PARSERUTILS_OK);

			if (strncmp((char *) cptr, "-->", SLEN("-->")) == 0) {
				tokeniser->escape_flag = false;
			}

			tokeniser->context.pending += len;
		} else if (c == '\0') {
			if (tokeniser->context.pending > 0) {
				/* Emit any pending characters */
				emit_current_chars(tokeniser);
			}

			/* Emit a replacement character */
			emit_character_token(tokeniser, &u_fffd_str);

			/* Advance past NUL */
			parserutils_inputstream_advance(tokeniser->input, 1);
		} else if (c == '\r') {
			error = parserutils_inputstream_peek(
					tokeniser->input,
					tokeniser->context.pending + len,
					&cptr,
					&len);

			if (error != PARSERUTILS_OK && 
					error != PARSERUTILS_EOF) {
				break;
			}

			if (tokeniser->context.pending > 0) {
				/* Emit any pending characters */
				emit_current_chars(tokeniser);
			}

			if (error == PARSERUTILS_EOF ||	*cptr != '\n') {
				/* Emit newline */
				emit_character_token(tokeniser, &lf_str);
			}

			/* Advance over */
			parserutils_inputstream_advance(tokeniser->input, 1);
		} else {
			/* Just collect into buffer */
			tokeniser->context.pending += len;
		}
	}

	if (tokeniser->state != STATE_TAG_OPEN &&
		(tokeniser->state != STATE_DATA || error == PARSERUTILS_EOF) &&
			tokeniser->context.pending > 0) {
		/* Emit any pending characters */
		emit_current_chars(tokeniser);
	}

	if (error == PARSERUTILS_EOF) {
		token.type = HUBBUB_TOKEN_EOF;
		hubbub_tokeniser_emit_token(tokeniser, &token);
	}

	if (error == PARSERUTILS_EOF) {
		return HUBBUB_NEEDDATA;
	} else {
		return hubbub_error_from_parserutils_error(error);
	}
}

/* emit any pending tokens before calling */
hubbub_error hubbub_tokeniser_handle_character_reference_data(
		hubbub_tokeniser *tokeniser)
{
	assert(tokeniser->context.pending == 0);

	if (tokeniser->context.match_entity.complete == false) {
		return hubbub_tokeniser_consume_character_reference(tokeniser,
				tokeniser->context.pending);
	} else {
		hubbub_token token;

		uint8_t utf8[6];
		uint8_t *utf8ptr = utf8;
		size_t len = sizeof(utf8);

		token.type = HUBBUB_TOKEN_CHARACTER;

		if (tokeniser->context.match_entity.codepoint) {
			parserutils_charset_utf8_from_ucs4(
				tokeniser->context.match_entity.codepoint,
				&utf8ptr, &len);

			token.data.character.ptr = utf8;
			token.data.character.len = sizeof(utf8) - len;

			hubbub_tokeniser_emit_token(tokeniser, &token);

			/* +1 for ampersand */
			parserutils_inputstream_advance(tokeniser->input,
					tokeniser->context.match_entity.length
							+ 1);
		} else {
			parserutils_error error;
			const uint8_t *cptr = NULL;
			error = parserutils_inputstream_peek(
					tokeniser->input,
					tokeniser->context.pending,
					&cptr,
					&len);

			assert(error == PARSERUTILS_OK);

			token.data.character.ptr = cptr;
			token.data.character.len = len;

			hubbub_tokeniser_emit_token(tokeniser, &token);
			parserutils_inputstream_advance(tokeniser->input, len);
		}

		/* Reset for next time */
		tokeniser->context.match_entity.complete = false;

		tokeniser->state = STATE_DATA;
	}

	return HUBBUB_OK;
}

/* this state always switches to another state straight away */
/* this state expects the current character to be '<' */
hubbub_error hubbub_tokeniser_handle_tag_open(hubbub_tokeniser *tokeniser)
{
	hubbub_tag *ctag = &tokeniser->context.current_tag;

	size_t len;
	const uint8_t *cptr;
	parserutils_error error;
	uint8_t c;

	assert(tokeniser->context.pending == 1);
/*	assert(tokeniser->context.chars.ptr[0] == '<'); */

	error = parserutils_inputstream_peek(tokeniser->input, 
			tokeniser->context.pending, &cptr, &len);

	if (error != PARSERUTILS_OK) {
		if (error == PARSERUTILS_EOF) {
			/* Return to data state with '<' still in "chars" */
			tokeniser->state = STATE_DATA;
			return HUBBUB_OK;
		} else {
			return hubbub_error_from_parserutils_error(error);
		}
	}

	c = *cptr;

	if (c == '/') {
		tokeniser->context.pending += len;

		tokeniser->context.close_tag_match.match = false;
		tokeniser->context.close_tag_match.count = 0;

		tokeniser->state = STATE_CLOSE_TAG_OPEN;
	} else if (tokeniser->content_model == HUBBUB_CONTENT_MODEL_RCDATA ||
			tokeniser->content_model ==
					HUBBUB_CONTENT_MODEL_CDATA) {
		/* Return to data state with '<' still in "chars" */
		tokeniser->state = STATE_DATA;
	} else if (tokeniser->content_model == HUBBUB_CONTENT_MODEL_PCDATA) {
		if (c == '!') {
			parserutils_inputstream_advance(tokeniser->input,
					SLEN("<!"));

			tokeniser->context.pending = 0;
			tokeniser->state = STATE_MARKUP_DECLARATION_OPEN;
		} else if ('A' <= c && c <= 'Z') {
			uint8_t lc = (c + 0x20);

			START_BUF(ctag->name, &lc, len);
			ctag->n_attributes = 0;
			tokeniser->context.current_tag_type =
					HUBBUB_TOKEN_START_TAG;

			tokeniser->context.pending += len;

			tokeniser->state = STATE_TAG_NAME;
		} else if ('a' <= c && c <= 'z') {
			START_BUF(ctag->name, cptr, len);
			ctag->n_attributes = 0;
			tokeniser->context.current_tag_type =
					HUBBUB_TOKEN_START_TAG;

			tokeniser->context.pending += len;

			tokeniser->state = STATE_TAG_NAME;
		} else if (c == '>') {
			/** \todo parse error */

			tokeniser->context.pending += len;
			tokeniser->state = STATE_DATA;
		} else if (c == '?') {
			/** \todo parse error */

			/* Cursor still at "<", need to advance past it */
			parserutils_inputstream_advance(
					tokeniser->input, SLEN("<"));
			tokeniser->context.pending = 0;

			tokeniser->state = STATE_BOGUS_COMMENT;
		} else {
			/* Return to data state with '<' still in "chars" */
			tokeniser->state = STATE_DATA;
		}
	}

	return HUBBUB_OK;
}

/* this state expects tokeniser->context.chars to be "</" */
/* this state never stays in this state for more than one character */
hubbub_error hubbub_tokeniser_handle_close_tag_open(hubbub_tokeniser *tokeniser)
{
	hubbub_tokeniser_context *ctx = &tokeniser->context;

	size_t len;
	const uint8_t *cptr;
	parserutils_error error;
	uint8_t c;

	assert(tokeniser->context.pending == 2);
/*	assert(tokeniser->context.chars.ptr[0] == '<'); */
/*	assert(tokeniser->context.chars.ptr[1] == '/'); */

	/**\todo fragment case */

	if (tokeniser->content_model == HUBBUB_CONTENT_MODEL_RCDATA ||
			tokeniser->content_model ==
					HUBBUB_CONTENT_MODEL_CDATA) {
		uint8_t *start_tag_name =
			tokeniser->context.last_start_tag_name;
		size_t start_tag_len =
			tokeniser->context.last_start_tag_len;

		while ((error = parserutils_inputstream_peek(tokeniser->input,
					ctx->pending +
						ctx->close_tag_match.count,
					&cptr,
					&len)) == PARSERUTILS_OK) {
			c = *cptr;

			if ((start_tag_name[ctx->close_tag_match.count] & ~0x20)
					!= (c & ~0x20)) {
				break;
			}

			ctx->close_tag_match.count += len;

			if (ctx->close_tag_match.count == start_tag_len) {
				ctx->close_tag_match.match = true;
				break;
			}
		}

		if (error != PARSERUTILS_OK && error != PARSERUTILS_EOF) {
			return hubbub_error_from_parserutils_error(error);
		}

		if (ctx->close_tag_match.match == true) {
			error = parserutils_inputstream_peek(
			 		tokeniser->input,
			 		ctx->pending +
				 		ctx->close_tag_match.count,
					&cptr,
			 		&len);

			if (error != PARSERUTILS_OK && 
					error != PARSERUTILS_EOF) {
				return hubbub_error_from_parserutils_error(
						error);
			} else if (error != PARSERUTILS_EOF) {
				c = *cptr;

				if (c != '\t' && c != '\n' && c != '\f' &&
						c != ' ' && c != '>' &&
						c != '/') {
					ctx->close_tag_match.match = false;
				}
			}
		}
	}

	if (ctx->close_tag_match.match == false &&
			tokeniser->content_model !=
					HUBBUB_CONTENT_MODEL_PCDATA) {
		/* We should emit "</" here, but instead we leave it in the
		 * buffer so the data state emits it with any characters
		 * following it */
		tokeniser->state = STATE_DATA;
	} else {
		error = parserutils_inputstream_peek(tokeniser->input,
				tokeniser->context.pending, &cptr, &len);

		if (error == PARSERUTILS_EOF) {
			/** \todo parse error */

			/* Return to data state with "</" pending */
			tokeniser->state = STATE_DATA;
			return HUBBUB_OK;
		} else if (error != PARSERUTILS_OK) {
			return hubbub_error_from_parserutils_error(error);
		}

		c = *cptr;

		if ('A' <= c && c <= 'Z') {
			uint8_t lc = (c + 0x20);
			START_BUF(tokeniser->context.current_tag.name,
					&lc, len);
			tokeniser->context.current_tag.n_attributes = 0;

			tokeniser->context.current_tag_type =
					HUBBUB_TOKEN_END_TAG;

			tokeniser->context.pending += len;

			tokeniser->state = STATE_TAG_NAME;
		} else if ('a' <= c && c <= 'z') {
			START_BUF(tokeniser->context.current_tag.name,
					cptr, len);
			tokeniser->context.current_tag.n_attributes = 0;

			tokeniser->context.current_tag_type =
					HUBBUB_TOKEN_END_TAG;

			tokeniser->context.pending += len;

			tokeniser->state = STATE_TAG_NAME;
		} else if (c == '>') {
			/* Cursor still at "</", need to collect ">" */
			tokeniser->context.pending += len;

			/* Now need to advance past "</>" */
			parserutils_inputstream_advance(tokeniser->input,
					tokeniser->context.pending);
			tokeniser->context.pending = 0;

			/** \todo parse error */
			tokeniser->state = STATE_DATA;
		} else {
			/** \todo parse error */

			/* Cursor still at "</", need to advance past it */
			parserutils_inputstream_advance(tokeniser->input,
					tokeniser->context.pending);
			tokeniser->context.pending = 0;

			tokeniser->state = STATE_BOGUS_COMMENT;
		}
	}

	return HUBBUB_OK;
}

/* this state expects tokeniser->context.current_tag to already have its
   first character set */
hubbub_error hubbub_tokeniser_handle_tag_name(hubbub_tokeniser *tokeniser)
{
	hubbub_tag *ctag = &tokeniser->context.current_tag;

	size_t len;
	const uint8_t *cptr;
	parserutils_error error;
	uint8_t c;

	assert(tokeniser->context.pending > 0);
/*	assert(tokeniser->context.chars.ptr[0] == '<'); */
	assert(ctag->name.len > 0);
/*	assert(ctag->name.ptr); */

	error = parserutils_inputstream_peek(tokeniser->input, 
			tokeniser->context.pending, &cptr, &len);

	if (error != PARSERUTILS_OK) {
		if (error == PARSERUTILS_EOF) {
			tokeniser->state = STATE_DATA;
			return emit_current_tag(tokeniser);
		} else {
			return hubbub_error_from_parserutils_error(error);
		}
	}

	c = *cptr;

	if (c == '\t' || c == '\n' || c == '\f' || c == ' ' || c == '\r') {
		tokeniser->context.pending += len;
		tokeniser->state = STATE_BEFORE_ATTRIBUTE_NAME;
	} else if (c == '>') {
		tokeniser->context.pending += len;
		tokeniser->state = STATE_DATA;
		return emit_current_tag(tokeniser);
	} else if (c == '\0') {
		COLLECT(ctag->name, u_fffd, sizeof(u_fffd));
		tokeniser->context.pending += len;
	} else if (c == '/') {
		tokeniser->context.pending += len;
		tokeniser->state = STATE_SELF_CLOSING_START_TAG;
	} else if ('A' <= c && c <= 'Z') {
		uint8_t lc = (c + 0x20);
		COLLECT(ctag->name, &lc, len);
		tokeniser->context.pending += len;
	} else {
		COLLECT(ctag->name, cptr, len);
		tokeniser->context.pending += len;
	}

	return HUBBUB_OK;
}

hubbub_error hubbub_tokeniser_handle_before_attribute_name(
		hubbub_tokeniser *tokeniser)
{
	hubbub_tag *ctag = &tokeniser->context.current_tag;

	size_t len;
	const uint8_t *cptr;
	parserutils_error error;
	uint8_t c;

	error = parserutils_inputstream_peek(tokeniser->input, 
			tokeniser->context.pending, &cptr, &len);

	if (error != PARSERUTILS_OK) {
		if (error == PARSERUTILS_EOF) {
			tokeniser->state = STATE_DATA;
			return emit_current_tag(tokeniser);
		} else {
			return hubbub_error_from_parserutils_error(error);
		}
	}

	c = *cptr;

	if (c == '\t' || c == '\n' || c == '\f' || c == ' ' || c == '\r') {
		/* pass over in silence */
		tokeniser->context.pending += len;
	} else if (c == '>') {
		tokeniser->context.pending += len;
		tokeniser->state = STATE_DATA;
		return emit_current_tag(tokeniser);
	} else if (c == '/') {
		tokeniser->context.pending += len;
		tokeniser->state = STATE_SELF_CLOSING_START_TAG;
	} else {
		hubbub_attribute *attr;

		if (c == '"' || c == '\'' || c == '=') {
			/** \todo parse error */
		}

		attr = tokeniser->alloc(ctag->attributes,
				(ctag->n_attributes + 1) *
					sizeof(hubbub_attribute),
				tokeniser->alloc_pw);
		if (attr == NULL)
			return HUBBUB_NOMEM;

		ctag->attributes = attr;

		if ('A' <= c && c <= 'Z') {
			uint8_t lc = (c + 0x20);
			START_BUF(attr[ctag->n_attributes].name, &lc, len);
		} else if (c == '\0') {
			START_BUF(attr[ctag->n_attributes].name,
					u_fffd, sizeof(u_fffd));
		} else {
			START_BUF(attr[ctag->n_attributes].name, cptr, len);
		}

		attr[ctag->n_attributes].ns = HUBBUB_NS_NULL;
		attr[ctag->n_attributes].value.ptr = NULL;
		attr[ctag->n_attributes].value.len = 0;

		ctag->n_attributes++;

		tokeniser->context.pending += len;
		tokeniser->state = STATE_ATTRIBUTE_NAME;
	}

	return HUBBUB_OK;
}

hubbub_error hubbub_tokeniser_handle_attribute_name(hubbub_tokeniser *tokeniser)
{
	hubbub_tag *ctag = &tokeniser->context.current_tag;

	size_t len;
	const uint8_t *cptr;
	parserutils_error error;
	uint8_t c;

	assert(ctag->attributes[ctag->n_attributes - 1].name.len > 0);

	error = parserutils_inputstream_peek(tokeniser->input, 
			tokeniser->context.pending, &cptr, &len);

	if (error != PARSERUTILS_OK) {
		if (error == PARSERUTILS_EOF) {
			tokeniser->state = STATE_DATA;
			return emit_current_tag(tokeniser);
		} else {
			return hubbub_error_from_parserutils_error(error);
		}
	}

	c = *cptr;

	if (c == '\t' || c == '\n' || c == '\f' || c == ' ' || c == '\r') {
		tokeniser->context.pending += len;
		tokeniser->state = STATE_AFTER_ATTRIBUTE_NAME;
	} else if (c == '=') {
		tokeniser->context.pending += len;
		tokeniser->state = STATE_BEFORE_ATTRIBUTE_VALUE;
	} else if (c == '>') {
		tokeniser->context.pending += len;
		tokeniser->state = STATE_DATA;
		return emit_current_tag(tokeniser);
	} else if (c == '/') {
		tokeniser->context.pending += len;
		tokeniser->state = STATE_SELF_CLOSING_START_TAG;
	} else if (c == '\0') {
		COLLECT(ctag->attributes[ctag->n_attributes - 1].name,
				u_fffd, sizeof(u_fffd));
		tokeniser->context.pending += len;
	} else if ('A' <= c && c <= 'Z') {
		uint8_t lc = (c + 0x20);
		COLLECT(ctag->attributes[ctag->n_attributes - 1].name,
				&lc, len);
		tokeniser->context.pending += len;
	} else {
		COLLECT(ctag->attributes[ctag->n_attributes - 1].name,
				cptr, len);
		tokeniser->context.pending += len;
	}

	return HUBBUB_OK;
}

hubbub_error hubbub_tokeniser_handle_after_attribute_name(
		hubbub_tokeniser *tokeniser)
{
	hubbub_tag *ctag = &tokeniser->context.current_tag;

	size_t len;
	const uint8_t *cptr;
	parserutils_error error;
	uint8_t c;

	error = parserutils_inputstream_peek(tokeniser->input, 
			tokeniser->context.pending, &cptr, &len);

	if (error != PARSERUTILS_OK) {
		if (error == PARSERUTILS_EOF) {
			tokeniser->state = STATE_DATA;
			return emit_current_tag(tokeniser);
		} else {
			return hubbub_error_from_parserutils_error(error);
		}
	}

	c = *cptr;

	if (c == '\t' || c == '\n' || c == '\f' || c == ' ' || c == '\r') {
		tokeniser->context.pending += len;
	} else if (c == '=') {
		tokeniser->context.pending += len;
		tokeniser->state = STATE_BEFORE_ATTRIBUTE_VALUE;
	} else if (c == '>') {
		tokeniser->context.pending += len;

		tokeniser->state = STATE_DATA;
		return emit_current_tag(tokeniser);
	} else if (c == '/') {
		tokeniser->context.pending += len;
		tokeniser->state = STATE_SELF_CLOSING_START_TAG;
	} else {
		hubbub_attribute *attr;

		if (c == '"' || c == '\'') {
			/** \todo parse error */
		}

		attr = tokeniser->alloc(ctag->attributes,
				(ctag->n_attributes + 1) *
					sizeof(hubbub_attribute),
				tokeniser->alloc_pw);
		if (attr == NULL)
			return HUBBUB_NOMEM;

		ctag->attributes = attr;

		if ('A' <= c && c <= 'Z') {
			uint8_t lc = (c + 0x20);
			START_BUF(attr[ctag->n_attributes].name, &lc, len);
		} else if (c == '\0') {
			START_BUF(attr[ctag->n_attributes].name,
					u_fffd, sizeof(u_fffd));
		} else {
			START_BUF(attr[ctag->n_attributes].name, cptr, len);
		}

		attr[ctag->n_attributes].ns = HUBBUB_NS_NULL;
		attr[ctag->n_attributes].value.ptr = NULL;
		attr[ctag->n_attributes].value.len = 0;

		ctag->n_attributes++;

		tokeniser->context.pending += len;
		tokeniser->state = STATE_ATTRIBUTE_NAME;
	}

	return HUBBUB_OK;
}

/* this state is only ever triggered by an '=' */
hubbub_error hubbub_tokeniser_handle_before_attribute_value(
		hubbub_tokeniser *tokeniser)
{
	hubbub_tag *ctag = &tokeniser->context.current_tag;

	size_t len;
	const uint8_t *cptr;
	parserutils_error error;
	uint8_t c;

	error = parserutils_inputstream_peek(tokeniser->input, 
			tokeniser->context.pending, &cptr, &len);

	if (error != PARSERUTILS_OK) {
		if (error == PARSERUTILS_EOF) {
			/** \todo parse error */
			tokeniser->state = STATE_DATA;
			return emit_current_tag(tokeniser);
		} else {
			return hubbub_error_from_parserutils_error(error);
		}
	}

	c = *cptr;

	if (c == '\t' || c == '\n' || c == '\f' || c == ' ' || c == '\r') {
		tokeniser->context.pending += len;
	} else if (c == '"') {
		tokeniser->context.pending += len;
		tokeniser->state = STATE_ATTRIBUTE_VALUE_DQ;
	} else if (c == '&') {
		/* Don't consume the '&' -- reprocess in UQ state */
		tokeniser->state = STATE_ATTRIBUTE_VALUE_UQ;
	} else if (c == '\'') {
		tokeniser->context.pending += len;
		tokeniser->state = STATE_ATTRIBUTE_VALUE_SQ;
	} else if (c == '>') {
		/** \todo parse error */
		tokeniser->context.pending += len;

		tokeniser->state = STATE_DATA;
		return emit_current_tag(tokeniser);
	} else if (c == '\0') {
		START_BUF(ctag->attributes[ctag->n_attributes - 1].value,
				u_fffd, sizeof(u_fffd));
		tokeniser->context.pending += len;
		tokeniser->state = STATE_ATTRIBUTE_VALUE_UQ;
	} else {
		if (c == '=') {
			/** \todo parse error */
		}

		START_BUF(ctag->attributes[ctag->n_attributes - 1].value,
				cptr, len);

		tokeniser->context.pending += len;
		tokeniser->state = STATE_ATTRIBUTE_VALUE_UQ;
	}

	return HUBBUB_OK;
}

hubbub_error hubbub_tokeniser_handle_attribute_value_dq(
		hubbub_tokeniser *tokeniser)
{
	hubbub_tag *ctag = &tokeniser->context.current_tag;

	size_t len;
	const uint8_t *cptr;
	parserutils_error error;
	uint8_t c;

	error = parserutils_inputstream_peek(tokeniser->input, 
			tokeniser->context.pending, &cptr, &len);

	if (error != PARSERUTILS_OK) {
		if (error == PARSERUTILS_EOF) {
			tokeniser->state = STATE_DATA;
			return emit_current_tag(tokeniser);
		} else {
			return hubbub_error_from_parserutils_error(error);
		}
	}

	c = *cptr;

	if (c == '"') {
		tokeniser->context.pending += len;
		tokeniser->state = STATE_AFTER_ATTRIBUTE_VALUE_Q;
	} else if (c == '&') {
		tokeniser->context.prev_state = tokeniser->state;
		tokeniser->state = STATE_CHARACTER_REFERENCE_IN_ATTRIBUTE_VALUE;
		tokeniser->context.allowed_char = '"';
		/* Don't eat the '&'; it'll be handled by entity consumption */
	} else if (c == '\0') {
		COLLECT_MS(ctag->attributes[ctag->n_attributes - 1].value,
				u_fffd, sizeof(u_fffd));
		tokeniser->context.pending += len;
	} else if (c == '\r') {
		error = parserutils_inputstream_peek(
				tokeniser->input,
				tokeniser->context.pending + len,
				&cptr,
				&len);

		if (error != PARSERUTILS_OK && error != PARSERUTILS_EOF) {
			return hubbub_error_from_parserutils_error(error);
		} else if (error == PARSERUTILS_EOF || *cptr != '\n') {
			COLLECT_MS(ctag->attributes[
					ctag->n_attributes - 1].value,
					&lf, sizeof(lf));
		}

		/* Consume '\r' */
		tokeniser->context.pending += 1;
	} else {
		COLLECT_MS(ctag->attributes[ctag->n_attributes - 1].value,
				cptr, len);
		tokeniser->context.pending += len;
	}

	return HUBBUB_OK;
}

hubbub_error hubbub_tokeniser_handle_attribute_value_sq(
		hubbub_tokeniser *tokeniser)
{
	hubbub_tag *ctag = &tokeniser->context.current_tag;

	size_t len;
	const uint8_t *cptr;
	parserutils_error error;
	uint8_t c;

	error = parserutils_inputstream_peek(tokeniser->input, 
			tokeniser->context.pending, &cptr, &len);

	if (error != PARSERUTILS_OK) {
		if (error == PARSERUTILS_EOF) {
			tokeniser->state = STATE_DATA;
			return emit_current_tag(tokeniser);
		} else {
			return hubbub_error_from_parserutils_error(error);
		}
	}

	c = *cptr;

	if (c == '\'') {
		tokeniser->context.pending += len;
		tokeniser->state = STATE_AFTER_ATTRIBUTE_VALUE_Q;
	} else if (c == '&') {
		tokeniser->context.prev_state = tokeniser->state;
		tokeniser->state = STATE_CHARACTER_REFERENCE_IN_ATTRIBUTE_VALUE;
		tokeniser->context.allowed_char = '\'';
		/* Don't eat the '&'; it'll be handled by entity consumption */
	} else if (c == '\0') {
		COLLECT_MS(ctag->attributes[ctag->n_attributes - 1].value,
				u_fffd, sizeof(u_fffd));
		tokeniser->context.pending += len;
	} else if (c == '\r') {
		error = parserutils_inputstream_peek(
				tokeniser->input,
				tokeniser->context.pending + len,
				&cptr,
				&len);

		if (error != PARSERUTILS_OK && error != PARSERUTILS_EOF) {
			return hubbub_error_from_parserutils_error(error);
		} else if (error == PARSERUTILS_EOF || *cptr != '\n') {
			COLLECT_MS(ctag->attributes[
					ctag->n_attributes - 1].value,
					&lf, sizeof(lf));
		}

		/* Consume \r */
		tokeniser->context.pending += 1;
	} else {
		COLLECT_MS(ctag->attributes[ctag->n_attributes - 1].value,
				cptr, len);
		tokeniser->context.pending += len;
	}

	return HUBBUB_OK;
}

hubbub_error hubbub_tokeniser_handle_attribute_value_uq(
		hubbub_tokeniser *tokeniser)
{
	hubbub_tag *ctag = &tokeniser->context.current_tag;
	uint8_t c;

	size_t len;
	const uint8_t *cptr;
	parserutils_error error;

	error = parserutils_inputstream_peek(tokeniser->input, 
			tokeniser->context.pending, &cptr, &len);

	if (error != PARSERUTILS_OK) {
		if (error == PARSERUTILS_EOF) {
			tokeniser->state = STATE_DATA;
			return emit_current_tag(tokeniser);
		} else {
			return hubbub_error_from_parserutils_error(error);
		}
	}

	c = *cptr;

	assert(c == '&' ||
		ctag->attributes[ctag->n_attributes - 1].value.len >= 1);

	if (c == '\t' || c == '\n' || c == '\f' || c == ' ' || c == '\r') {
		tokeniser->context.pending += len;
		tokeniser->state = STATE_BEFORE_ATTRIBUTE_NAME;
	} else if (c == '&') {
		tokeniser->context.prev_state = tokeniser->state;
		tokeniser->state = STATE_CHARACTER_REFERENCE_IN_ATTRIBUTE_VALUE;
		/* Don't eat the '&'; it'll be handled by entity consumption */
	} else if (c == '>') {
		tokeniser->context.pending += len;
		tokeniser->state = STATE_DATA;
		return emit_current_tag(tokeniser);
	} else if (c == '\0') {
		COLLECT(ctag->attributes[ctag->n_attributes - 1].value,
				u_fffd, sizeof(u_fffd));
		tokeniser->context.pending += len;
	} else {
		if (c == '"' || c == '\'' || c == '=') {
			/** \todo parse error */
		}

		COLLECT(ctag->attributes[ctag->n_attributes - 1].value,
				cptr, len);
		tokeniser->context.pending += len;
	}

	return HUBBUB_OK;
}

hubbub_error hubbub_tokeniser_handle_character_reference_in_attribute_value(
		hubbub_tokeniser *tokeniser)
{
	if (tokeniser->context.match_entity.complete == false) {
		return hubbub_tokeniser_consume_character_reference(tokeniser,
				tokeniser->context.pending);
	} else {
		hubbub_tag *ctag = &tokeniser->context.current_tag;
		hubbub_attribute *attr = &ctag->attributes[
				ctag->n_attributes - 1];

		uint8_t utf8[6];
		uint8_t *utf8ptr = utf8;
		size_t len = sizeof(utf8);

		if (tokeniser->context.match_entity.codepoint) {
			parserutils_charset_utf8_from_ucs4(
				tokeniser->context.match_entity.codepoint,
				&utf8ptr, &len);

			COLLECT_MS(attr->value, utf8, sizeof(utf8) - len);

			/* +1 for the ampersand */
			tokeniser->context.pending +=
					tokeniser->context.match_entity.length
					+ 1;
		} else {
			size_t len = 0;
			const uint8_t *cptr = NULL;
			parserutils_error error;

			error = parserutils_inputstream_peek(
					tokeniser->input,
					tokeniser->context.pending, 
					&cptr,
					&len);

			assert(error == PARSERUTILS_OK);

			/* Insert the ampersand */
			COLLECT_MS(attr->value, cptr, len);
			tokeniser->context.pending += len;
		}

		/* Reset for next time */
		tokeniser->context.match_entity.complete = false;

		/* And back to the previous state */
		tokeniser->state = tokeniser->context.prev_state;
	}

	return HUBBUB_OK;
}

/* always switches state */
hubbub_error hubbub_tokeniser_handle_after_attribute_value_q(
		hubbub_tokeniser *tokeniser)
{
	size_t len;
	const uint8_t *cptr;
	parserutils_error error;
	uint8_t c;

	error = parserutils_inputstream_peek(tokeniser->input, 
			tokeniser->context.pending, &cptr, &len);

	if (error != PARSERUTILS_OK) {
		if (error == PARSERUTILS_EOF) {
			tokeniser->state = STATE_DATA;
			return emit_current_tag(tokeniser);
		} else {
			return hubbub_error_from_parserutils_error(error);
		}
	}

	c = *cptr;

	if (c == '\t' || c == '\n' || c == '\f' || c == ' ' || c == '\r') {
		tokeniser->context.pending += len;
		tokeniser->state = STATE_BEFORE_ATTRIBUTE_NAME;
	} else if (c == '>') {
		tokeniser->context.pending += len;

		tokeniser->state = STATE_DATA;
		return emit_current_tag(tokeniser);
	} else if (c == '/') {
		tokeniser->context.pending += len;
		tokeniser->state = STATE_SELF_CLOSING_START_TAG;
	} else {
		/** \todo parse error */
		/* Reprocess character in before attribute name state */
		tokeniser->state = STATE_BEFORE_ATTRIBUTE_NAME;
	}

	return HUBBUB_OK;
}

hubbub_error hubbub_tokeniser_handle_self_closing_start_tag(
		hubbub_tokeniser *tokeniser)
{
	size_t len;
	const uint8_t *cptr;
	parserutils_error error;
	uint8_t c;

	error = parserutils_inputstream_peek(tokeniser->input,
			tokeniser->context.pending, &cptr, &len);

	if (error != PARSERUTILS_OK) {
		if (error == PARSERUTILS_EOF) {
			tokeniser->state = STATE_DATA;
			return emit_current_tag(tokeniser);
		} else {
			return hubbub_error_from_parserutils_error(error);
		}
	}

	c = *cptr;

	if (c == '>') {
		tokeniser->context.pending += len;
		tokeniser->state = STATE_DATA;

		tokeniser->context.current_tag.self_closing = true;
		return emit_current_tag(tokeniser);
	} else {
		/* Reprocess character in before attribute name state */
		tokeniser->state = STATE_BEFORE_ATTRIBUTE_NAME;
	}

	return HUBBUB_OK;
}

/* this state expects tokeniser->context.chars to be empty on first entry */
hubbub_error hubbub_tokeniser_handle_bogus_comment(hubbub_tokeniser *tokeniser)
{
	size_t len;
	const uint8_t *cptr;
	parserutils_error error;
	uint8_t c;

	error = parserutils_inputstream_peek(tokeniser->input,
			tokeniser->context.pending, &cptr, &len);

	if (error != PARSERUTILS_OK) {
		if (error == PARSERUTILS_EOF) {
			tokeniser->state = STATE_DATA;
			return emit_current_comment(tokeniser);
		} else {
			return hubbub_error_from_parserutils_error(error);
		}
	}

	c = *cptr;

	if (c == '>') {
		tokeniser->context.pending += len;
		tokeniser->state = STATE_DATA;
		return emit_current_comment(tokeniser);
	} else if (c == '\0') {
		error = parserutils_buffer_append(tokeniser->buffer,
				u_fffd, sizeof(u_fffd));
		if (error != PARSERUTILS_OK)
			return hubbub_error_from_parserutils_error(error);

		tokeniser->context.pending += len;
	} else if (c == '\r') {
		error = parserutils_inputstream_peek(
				tokeniser->input,
				tokeniser->context.pending,
				&cptr,
				&len);

		if (error != PARSERUTILS_OK && error != PARSERUTILS_EOF) {
			return hubbub_error_from_parserutils_error(error);
		} else if (error == PARSERUTILS_EOF || *cptr != '\n') {
			error = parserutils_buffer_append(tokeniser->buffer,
					&lf, sizeof(lf));
			if (error != PARSERUTILS_OK) {
				return hubbub_error_from_parserutils_error(
						error);
			}
		}
		tokeniser->context.pending += len;
	} else {
		error = parserutils_buffer_append(tokeniser->buffer,
				(uint8_t *) cptr, len);
		if (error != PARSERUTILS_OK)
			return hubbub_error_from_parserutils_error(error);

		tokeniser->context.pending += len;
	}

	return HUBBUB_OK;
}

/* this state always switches to another state straight away */
hubbub_error hubbub_tokeniser_handle_markup_declaration_open(
		hubbub_tokeniser *tokeniser)
{
	size_t len;
	const uint8_t *cptr;
	parserutils_error error;
	uint8_t c;

	assert(tokeniser->context.pending == 0);

	error = parserutils_inputstream_peek(tokeniser->input, 0, &cptr, &len);

	if (error != PARSERUTILS_OK) {
		if (error == PARSERUTILS_EOF) {
			tokeniser->state = STATE_BOGUS_COMMENT;
			return HUBBUB_OK;
		} else {
			return hubbub_error_from_parserutils_error(error);
		}
	}

	c = *cptr;

	if (c == '-') {
		tokeniser->context.pending = len;
		tokeniser->state = STATE_MATCH_COMMENT;
	} else if ((c & ~0x20) == 'D') {
		tokeniser->context.pending = len;
		tokeniser->context.match_doctype.count = len;
		tokeniser->state = STATE_MATCH_DOCTYPE;
	} else if (tokeniser->process_cdata_section == true && c == '[') {
		tokeniser->context.pending = len;
		tokeniser->context.match_cdata.count = len;
		tokeniser->state = STATE_MATCH_CDATA;
	} else {
		tokeniser->state = STATE_BOGUS_COMMENT;
	}

	return HUBBUB_OK;
}


hubbub_error hubbub_tokeniser_handle_match_comment(hubbub_tokeniser *tokeniser)
{
	size_t len;
	const uint8_t *cptr;
	parserutils_error error;

	error = parserutils_inputstream_peek(tokeniser->input, 
			tokeniser->context.pending, &cptr, &len);

	if (error != PARSERUTILS_OK) {
		if (error == PARSERUTI…