/****************************************************************************
**
** Copyright (C) 2012 Nokia Corporation and/or its subsidiary(-ies).
** All rights reserved.
** Contact: Nokia Corporation (qt-info@nokia.com)
**
** This file is part of the tools applications of the Qt Toolkit.
**
** $QT_BEGIN_LICENSE:LGPL$
** GNU Lesser General Public License Usage
** This file may be used under the terms of the GNU Lesser General Public
** License version 2.1 as published by the Free Software Foundation and
** appearing in the file LICENSE.LGPL included in the packaging of this
** file. Please review the following information to ensure the GNU Lesser
** General Public License version 2.1 requirements will be met:
** http://www.gnu.org/licenses/old-licenses/lgpl-2.1.html.
**
** In addition, as a special exception, Nokia gives you certain additional
** rights. These rights are described in the Nokia Qt LGPL Exception
** version 1.1, included in the file LGPL_EXCEPTION.txt in this package.
**
** GNU General Public License Usage
** Alternatively, this file may be used under the terms of the GNU General
** Public License version 3.0 as published by the Free Software Foundation
** and appearing in the file LICENSE.GPL included in the packaging of this
** file. Please review the following information to ensure the GNU General
** Public License version 3.0 requirements will be met:
** http://www.gnu.org/copyleft/gpl.html.
**
** Other Usage
** Alternatively, this file may be used in accordance with the terms and
** conditions contained in a signed written agreement between you and Nokia.
**
**
**
**
**
** $QT_END_LICENSE$
**
****************************************************************************/

#include "config.h"
#include "doc.h"
#include "codemarker.h"
#include "editdistance.h"
#include "openedlist.h"
#include "quoter.h"
#include "text.h"
#include "tokenizer.h"
#include <qdatetime.h>
#include <qfile.h>
#include <qfileinfo.h>
#include <qhash.h>
#include <qtextstream.h>
#include <qregexp.h>
#include <ctype.h>
#include <limits.h>
#include <qdebug.h>

QT_BEGIN_NAMESPACE

Q_GLOBAL_STATIC(QSet<QString>, null_Set_QString)
Q_GLOBAL_STATIC(QStringList, null_QStringList)
Q_GLOBAL_STATIC(QList<Text>, null_QList_Text)
//Q_GLOBAL_STATIC(QStringMap, null_QStringMap)
Q_GLOBAL_STATIC(QStringMultiMap, null_QStringMultiMap)

struct Macro
{
    QString defaultDef;
    Location defaultDefLocation;
    QStringMap otherDefs;
    int numParams;
};

enum {
    CMD_A,
    CMD_ABSTRACT,
    CMD_ANNOTATEDLIST,
    CMD_BADCODE,
    CMD_BASENAME,
    CMD_BOLD,
    CMD_BRIEF,
    CMD_C,
    CMD_CAPTION,
    CMD_CHAPTER, // 9
    CMD_CODE,
    CMD_CODELINE,
    CMD_DIV,
    CMD_DOTS,
    CMD_ELSE,
    CMD_ENDABSTRACT,
    CMD_ENDCHAPTER,
    CMD_ENDCODE,
    CMD_ENDDIV,
    CMD_ENDFOOTNOTE,
    CMD_ENDIF,
    CMD_ENDLEGALESE,
    CMD_ENDLINK,
    CMD_ENDLIST,
    CMD_ENDOMIT,
    CMD_ENDPART,
    CMD_ENDQUOTATION,
    CMD_ENDRAW,
    CMD_ENDSECTION1,
    CMD_ENDSECTION2,
    CMD_ENDSECTION3,
    CMD_ENDSECTION4,
    CMD_ENDSIDEBAR,
    CMD_ENDTABLE,
    CMD_EXPIRE,
    CMD_FOOTNOTE,
    CMD_GENERATELIST,
    CMD_GRANULARITY,
    CMD_HEADER,
    CMD_I,
    CMD_IF,
    CMD_IMAGE,
    CMD_INCLUDE,
    CMD_INLINEIMAGE,
    CMD_INDEX,
    CMD_KEYWORD,
    CMD_L,
    CMD_LEGALESE,
    CMD_LINK,
    CMD_LIST,
    CMD_META,
    CMD_NEWCODE,
    CMD_O,
    CMD_OLDCODE,
    CMD_OMIT,
    CMD_OMITVALUE,
    CMD_OVERLOAD,
    CMD_PART,
    CMD_PRINTLINE,
    CMD_PRINTTO,
    CMD_PRINTUNTIL,
    CMD_QUOTATION,
    CMD_QUOTEFILE,
    CMD_QUOTEFROMFILE,
    CMD_QUOTEFUNCTION,
    CMD_RAW,
    CMD_ROW,
    CMD_SA,
    CMD_SECTION1, // 68
    CMD_SECTION2, // 69
    CMD_SECTION3, // 70
    CMD_SECTION4, // 71
    CMD_SIDEBAR,
    CMD_SINCELIST,
    CMD_SKIPLINE,
    CMD_SKIPTO,
    CMD_SKIPUNTIL,
    CMD_SNIPPET,
    CMD_SPAN,
    CMD_SUB,
    CMD_SUP,
    CMD_TABLE,
    CMD_TABLEOFCONTENTS,
    CMD_TARGET,
    CMD_TT,
    CMD_UNDERLINE,
    CMD_UNICODE,
    CMD_VALUE,
    CMD_WARNING,
    CMD_QML,
    CMD_ENDQML,
    CMD_CPP,
    CMD_ENDCPP,
    CMD_QMLTEXT,
    CMD_ENDQMLTEXT,
    CMD_CPPTEXT,
    CMD_ENDCPPTEXT,
    CMD_JS,
    CMD_ENDJS,
    NOT_A_CMD
};

static struct {
    const char *english;
    int no;
    QString *alias;
} cmds[] = {
    { "a", CMD_A, 0 },
    { "abstract", CMD_ABSTRACT, 0 },
    { "annotatedlist", CMD_ANNOTATEDLIST, 0 },
    { "badcode", CMD_BADCODE, 0 },
    { "basename", CMD_BASENAME, 0 }, // ### don't document for now
    { "bold", CMD_BOLD, 0 },
    { "brief", CMD_BRIEF, 0 },
    { "c", CMD_C, 0 },
    { "caption", CMD_CAPTION, 0 },
    { "chapter", CMD_CHAPTER, 0 },
    { "code", CMD_CODE, 0 },
    { "codeline", CMD_CODELINE, 0},
    { "div", CMD_DIV, 0 },
    { "dots", CMD_DOTS, 0 },
    { "else", CMD_ELSE, 0 },
    { "endabstract", CMD_ENDABSTRACT, 0 },
    { "endchapter", CMD_ENDCHAPTER, 0 },
    { "endcode", CMD_ENDCODE, 0 },
    { "enddiv", CMD_ENDDIV, 0 },
    { "endfootnote", CMD_ENDFOOTNOTE, 0 },
    { "endif", CMD_ENDIF, 0 },
    { "endlegalese", CMD_ENDLEGALESE, 0 },
    { "endlink", CMD_ENDLINK, 0 },
    { "endlist", CMD_ENDLIST, 0 },
    { "endomit", CMD_ENDOMIT, 0 },
    { "endpart", CMD_ENDPART, 0 },
    { "endquotation", CMD_ENDQUOTATION, 0 },
    { "endraw", CMD_ENDRAW, 0 },
    { "endsection1", CMD_ENDSECTION1, 0 },  // ### don't document for now
    { "endsection2", CMD_ENDSECTION2, 0 },  // ### don't document for now
    { "endsection3", CMD_ENDSECTION3, 0 },  // ### don't document for now
    { "endsection4", CMD_ENDSECTION4, 0 },  // ### don't document for now
    { "endsidebar", CMD_ENDSIDEBAR, 0 },
    { "endtable", CMD_ENDTABLE, 0 },
    { "expire", CMD_EXPIRE, 0 },
    { "footnote", CMD_FOOTNOTE, 0 },
    { "generatelist", CMD_GENERATELIST, 0 },
    { "granularity", CMD_GRANULARITY, 0 }, // ### don't document for now
    { "header", CMD_HEADER, 0 },
    { "i", CMD_I, 0 },
    { "if", CMD_IF, 0 },
    { "image", CMD_IMAGE, 0 },
    { "include", CMD_INCLUDE, 0 },
    { "inlineimage", CMD_INLINEIMAGE, 0 },
    { "index", CMD_INDEX, 0 }, // ### don't document for now
    { "keyword", CMD_KEYWORD, 0 },
    { "l", CMD_L, 0 },
    { "legalese", CMD_LEGALESE, 0 },
    { "link", CMD_LINK, 0 },
    { "list", CMD_LIST, 0 },
    { "meta", CMD_META, 0 },
    { "newcode", CMD_NEWCODE, 0 },
    { "o", CMD_O, 0 },
    { "oldcode", CMD_OLDCODE, 0 },
    { "omit", CMD_OMIT, 0 },
    { "omitvalue", CMD_OMITVALUE, 0 },
    { "overload", CMD_OVERLOAD, 0 },
    { "part", CMD_PART, 0 },
    { "printline", CMD_PRINTLINE, 0 },
    { "printto", CMD_PRINTTO, 0 },
    { "printuntil", CMD_PRINTUNTIL, 0 },
    { "quotation", CMD_QUOTATION, 0 },
    { "quotefile", CMD_QUOTEFILE, 0 },
    { "quotefromfile", CMD_QUOTEFROMFILE, 0 },
    { "quotefunction", CMD_QUOTEFUNCTION, 0 }, // ### don't document for now
    { "raw", CMD_RAW, 0 },
    { "row", CMD_ROW, 0 },
    { "sa", CMD_SA, 0 },
    { "section1", CMD_SECTION1, 0 },
    { "section2", CMD_SECTION2, 0 },
    { "section3", CMD_SECTION3, 0 },
    { "section4", CMD_SECTION4, 0 },
    { "sidebar", CMD_SIDEBAR, 0 }, // ### don't document for now
    { "sincelist", CMD_SINCELIST, 0 },
    { "skipline", CMD_SKIPLINE, 0 },
    { "skipto", CMD_SKIPTO, 0 },
    { "skipuntil", CMD_SKIPUNTIL, 0 },
    { "snippet", CMD_SNIPPET, 0 },
    { "span", CMD_SPAN, 0 },
    { "sub", CMD_SUB, 0 },
    { "sup", CMD_SUP, 0 },
    { "table", CMD_TABLE, 0 },
    { "tableofcontents", CMD_TABLEOFCONTENTS, 0 },
    { "target", CMD_TARGET, 0 },
    { "tt", CMD_TT, 0 },
    { "underline", CMD_UNDERLINE, 0 },
    { "unicode", CMD_UNICODE, 0 },
    { "value", CMD_VALUE, 0 },
    { "warning", CMD_WARNING, 0 },
    { "qml", CMD_QML, 0 },
    { "endqml", CMD_ENDQML, 0 },
    { "cpp", CMD_CPP, 0 },
    { "endcpp", CMD_ENDCPP, 0 },
    { "qmltext", CMD_QMLTEXT, 0 },
    { "endqmltext", CMD_ENDQMLTEXT, 0 },
    { "cpptext", CMD_CPPTEXT, 0 },
    { "endcpptext", CMD_ENDCPPTEXT, 0 },
    { "js", CMD_JS, 0 },
    { "endjs", CMD_ENDJS, 0 },
    { 0, 0, 0 }
};

typedef QHash<QString, int> QHash_QString_int;
typedef QHash<QString, Macro> QHash_QString_Macro;

Q_GLOBAL_STATIC(QStringMap, aliasMap)
Q_GLOBAL_STATIC(QHash_QString_int, cmdHash)
Q_GLOBAL_STATIC(QHash_QString_Macro, macroHash)

class DocPrivateExtra
{
  public:
    QString             baseName;
    Doc::Sections       granularity;
    Doc::Sections       section; // ###
    QList<Atom*>        tableOfContents;
    QList<int>          tableOfContentsLevels;
    QList<Atom*>        keywords;
    QList<Atom*>        targets;
    QStringMultiMap     metaMap;

    DocPrivateExtra()
        : granularity(Doc::Part) { }
};

struct Shared // ### get rid of
{
    Shared()
        : count(1) { }
    void ref() { ++count; }
    bool deref() { return (--count == 0); }

    int count;
};

static QString cleanLink(const QString &link)
{
    int colonPos = link.indexOf(':');
    if ((colonPos == -1) ||
        (!link.startsWith("file:") && !link.startsWith("mailto:")))
        return link;
    return link.mid(colonPos + 1).simplified();
}

class DocPrivate : public Shared
{
  public:
    DocPrivate(const Location& start = Location::null,
               const Location& end = Location::null,
               const QString& source = "");
    ~DocPrivate();

    void addAlso(const Text& also);
    void constructExtra();
    bool isEnumDocSimplifiable() const;

    // ### move some of this in DocPrivateExtra
    Location start_loc;
    Location end_loc;
    QString src;
    Text text;
    QSet<QString> params;
    QList<Text> alsoList;
    QStringList enumItemList;
    QStringList omitEnumItemList;
    QSet<QString> metacommandsUsed;
    QCommandMap metaCommandMap;
    bool hasLegalese : 1;
    bool hasSectioningUnits : 1;
    DocPrivateExtra *extra;
};

DocPrivate::DocPrivate(const Location& start,
                       const Location& end,
                       const QString& source)
    : start_loc(start),
      end_loc(end),
      src(source),
      hasLegalese(false),
      hasSectioningUnits(false),
      extra(0)
{
    // nothing.
}

DocPrivate::~DocPrivate()
{
    delete extra;
}

void DocPrivate::addAlso(const Text& also)
{
    alsoList.append(also);
}

void DocPrivate::constructExtra()
{
    if (extra == 0)
        extra = new DocPrivateExtra;
}

bool DocPrivate::isEnumDocSimplifiable() const
{
    bool justMetColon = false;
    int numValueTables = 0;

    const Atom *atom = text.firstAtom();
    while (atom) {
        if (atom->type() == Atom::AutoLink || atom->type() == Atom::String) {
            justMetColon = atom->string().endsWith(":");
        }
        else if ((atom->type() == Atom::ListLeft) &&
                   (atom->string() == ATOM_LIST_VALUE)) {
            if (justMetColon || numValueTables > 0)
                return false;
            ++numValueTables;
        }
        atom = atom->next();
    }
    return true;
}

class DocParser
{
  public:
    void parse(const QString &source,
               DocPrivate *docPrivate,
               const QSet<QString> &metaCommandSet);

    static int endCmdFor(int cmd);
    static QString cmdName(int cmd);
    static QString endCmdName(int cmd);
    static QString untabifyEtc(const QString& str);
    static int indentLevel(const QString& str);
    static QString unindent(int level, const QString& str);
    static QString slashed(const QString& str);

    static int tabSize;
    static QStringList exampleFiles;
    static QStringList exampleDirs;
    static QStringList sourceFiles;
    static QStringList sourceDirs;
    static bool quoting;

  private:
    Location& location();
    QString detailsUnknownCommand(const QSet<QString>& metaCommandSet,
                                   const QString& str);
    void checkExpiry(const QString& date);
    void insertBaseName(const QString &baseName);
    void insertTarget(const QString& target, bool keyword);
    void include(const QString& fileName, const QString& identifier);
    void startFormat(const QString& format, int cmd);
    bool openCommand(int cmd);
    bool closeCommand(int endCmd);
    void startSection(Doc::Sections unit, int cmd);
    void endSection(int unit, int endCmd);
    void parseAlso();
    void append(Atom::Type type, const QString& string = "");
    void append(Atom::Type type, const QString& p1, const QString& p2);
    void appendChar(QChar ch);
    void appendWord(const QString &word);
    void appendToCode(const QString &code);
    void appendToCode(const QString &code, Atom::Type defaultType);
    void startNewPara();
    void enterPara(Atom::Type leftType = Atom::ParaLeft,
                   Atom::Type rightType = Atom::ParaRight,
                   const QString& string = "");
    void leavePara();
    void leaveValue();
    void leaveValueList();
    void leaveTableRow();
    CodeMarker *quoteFromFile();
    void expandMacro(const QString& name, const QString& def, int numParams);
    QString expandMacroToString(const QString &name, const QString &def, int numParams);
    Doc::Sections getSectioningUnit();
    QString getArgument(bool verbatim = false);
    QString getOptionalArgument();
    QString getRestOfLine();
    QString getMetaCommandArgument(const QString &cmdStr);
    QString getUntilEnd(int cmd);
    QString getCode(int cmd, CodeMarker *marker);
    QString getUnmarkedCode(int cmd);

    bool isBlankLine();
    bool isLeftBraceAhead();
    void skipSpacesOnLine();
    void skipSpacesOrOneEndl();
    void skipAllSpaces();
    void skipToNextPreprocessorCommand();

    QStack<int> openedInputs;

    QString in;
    int pos;
    int len;
    Location cachedLoc;
    int cachedPos;

    DocPrivate* priv;
    enum ParagraphState {
        OutsideParagraph,
        InSingleLineParagraph,
        InMultiLineParagraph
    };
    ParagraphState paraState;
    bool inTableHeader;
    bool inTableRow;
    bool inTableItem;
    bool indexStartedPara; // ### rename
    Atom::Type pendingParaLeftType;
    Atom::Type pendingParaRightType;
    QString pendingParaString;

    int braceDepth;
    int minIndent;
    Doc::Sections currentSection;
    QMap<QString, Location> targetMap;
    QMap<int, QString> pendingFormats;
    QStack<int> openedCommands;
    QStack<OpenedList> openedLists;
    Quoter quoter;
};

int DocParser::tabSize;
QStringList DocParser::exampleFiles;
QStringList DocParser::exampleDirs;
QStringList DocParser::sourceFiles;
QStringList DocParser::sourceDirs;
bool DocParser::quoting;

/*!
  Parse the \a source string to build a Text data structure
  in \a docPrivate. The Text data structure is a linked list
  of Atoms.

  \a metaCommandSet is the set of metacommands that may be
  found in \a source. These metacommands are not markup text
  commands. They are topic commands and related metacommands.
 */
void DocParser::parse(const QString& source,
                      DocPrivate *docPrivate,
                      const QSet<QString>& metaCommandSet)
{
    in = source;
    pos = 0;
    len = in.length();
    cachedLoc = docPrivate->start_loc;
    cachedPos = 0;
    priv = docPrivate;
    priv->text << Atom::Nop;

    paraState = OutsideParagraph;
    inTableHeader = false;
    inTableRow = false;
    inTableItem = false;
    indexStartedPara = false;
    pendingParaLeftType = Atom::Nop;
    pendingParaRightType = Atom::Nop;

    braceDepth = 0;
    minIndent = INT_MAX;
    currentSection = Doc::NoSection;
    openedCommands.push(CMD_OMIT);
    quoter.reset();

    CodeMarker *marker = 0;
    Atom *currentLinkAtom = 0;
    QString p1, p2;
    QStack<bool> preprocessorSkipping;
    int numPreprocessorSkipping = 0;

    while (pos < len) {
        QChar ch = in.at(pos);

        switch (ch.unicode()) {
        case '\\':
            {
                QString cmdStr;
                pos++;
                while (pos < len) {
                    ch = in.at(pos);
                    if (ch.isLetterOrNumber()) {
                        cmdStr += ch;
                        pos++;
                    }
                    else {
                        break;
                    }
                }
                if (cmdStr.isEmpty()) {
                    if (pos < len) {
                        enterPara();
                        if (in.at(pos).isSpace()) {
                            skipAllSpaces();
                            appendChar(QLatin1Char(' '));
                        }
                        else {
                            appendChar(in.at(pos++));
                        }
                    }
                }
                else {
                    int cmd = cmdHash()->value(cmdStr,NOT_A_CMD);
                    switch (cmd) {
                    case CMD_A:
                        enterPara();
                        p1 = getArgument();
                        append(Atom::FormattingLeft,ATOM_FORMATTING_PARAMETER);
                        append(Atom::String, p1);
                        append(Atom::FormattingRight,ATOM_FORMATTING_PARAMETER);
                        priv->params.insert(p1);
                        break;
                    case CMD_ABSTRACT:
                        if (openCommand(cmd)) {
                            leavePara();
                            append(Atom::AbstractLeft);
                        }
                        break;
                    case CMD_BADCODE:
                        leavePara();
                        append(Atom::CodeBad,getCode(CMD_BADCODE, marker));
                        break;
                    case CMD_BASENAME:
                        leavePara();
                        insertBaseName(getArgument());
                        break;
                    case CMD_BOLD:
                        startFormat(ATOM_FORMATTING_BOLD, cmd);
                        break;
                    case CMD_BRIEF:
                        leavePara();
                        enterPara(Atom::BriefLeft, Atom::BriefRight);
                        break;
                    case CMD_C:
                        enterPara();
                        p1 = untabifyEtc(getArgument(true));
                        marker = CodeMarker::markerForCode(p1);
                        append(Atom::C, marker->markedUpCode(p1, 0, location()));
                        break;
                    case CMD_CAPTION:
                        leavePara();
                        enterPara(Atom::CaptionLeft, Atom::CaptionRight);
                        break;
                    case CMD_CHAPTER:
                        startSection(Doc::Chapter, cmd);
                        break;
                    case CMD_CODE:
                        leavePara();
                        append(Atom::Code, getCode(CMD_CODE, 0));
                        break;
                    case CMD_QML:
                        leavePara();
                        append(Atom::Qml, getCode(CMD_QML, CodeMarker::markerForLanguage(QLatin1String("QML"))));
                        break;
                    case CMD_QMLTEXT:
                        append(Atom::QmlText);
                        break;
                    case CMD_JS:
                        leavePara();
                        append(Atom::JavaScript, getCode(CMD_JS, CodeMarker::markerForLanguage(QLatin1String("JavaScript"))));
                        break;
                    case CMD_DIV:
                        leavePara();
                        p1 = getArgument(true);
                        append(Atom::DivLeft, p1);
                        openedCommands.push(cmd);
                        break;
                    case CMD_ENDDIV:
                        leavePara();
                        append(Atom::DivRight);
                        closeCommand(cmd);
                        break;
                    case CMD_CODELINE:
                        {
                            if (!quoting) {
                                if (priv->text.lastAtom()->type() == Atom::Code
                                        && priv->text.lastAtom()->string().endsWith("\n\n"))
                                    priv->text.lastAtom()->chopString();
                                appendToCode("\n");
                            }
                            else {
                                append(Atom::CodeQuoteCommand, cmdStr);
                                append(Atom::CodeQuoteArgument, " ");
                            }
                        }
                        break;
                    case CMD_DOTS:
                        {
                            if (!quoting) {
                                if (priv->text.lastAtom()->type() == Atom::Code
                                        && priv->text.lastAtom()->string().endsWith("\n\n"))
                                    priv->text.lastAtom()->chopString();

                                QString arg = getOptionalArgument();
                                int indent = 4;
                                if (!arg.isEmpty())
                                    indent = arg.toInt();
                                for (int i = 0; i < indent; ++i)
                                    appendToCode(" ");
                                appendToCode("...\n");
                            }
                            else {
                                append(Atom::CodeQuoteCommand, cmdStr);
                                QString arg = getOptionalArgument();
                                if (arg.isEmpty())
                                    arg = "4";
                                append(Atom::CodeQuoteArgument, arg);
                            }
                        }
                        break;
                    case CMD_ELSE:
                        if (preprocessorSkipping.size() > 0) {
                            if (preprocessorSkipping.top()) {
                                --numPreprocessorSkipping;
                            }
                            else {
                                ++numPreprocessorSkipping;
                            }
                            preprocessorSkipping.top() = !preprocessorSkipping.top();
                            (void)getRestOfLine(); // ### should ensure that it's empty
                            if (numPreprocessorSkipping)
                                skipToNextPreprocessorCommand();
                        }
                        else {
                            location().warning(tr("Unexpected '\\%1'").arg(cmdName(CMD_ELSE)));
                        }
                        break;
                    case CMD_ENDABSTRACT:
                        if (closeCommand(cmd)) {
                            leavePara();
                            append(Atom::AbstractRight);
                        }
                        break;
                    case CMD_ENDCHAPTER:
                        endSection(Doc::Chapter, cmd);
                        break;
                    case CMD_ENDCODE:
                        closeCommand(cmd);
                        break;
                    case CMD_ENDQML:
                        closeCommand(cmd);
                        break;
                    case CMD_ENDQMLTEXT:
                        append(Atom::EndQmlText);
                        break;
                    case CMD_ENDJS:
                        closeCommand(cmd);
                        break;
                    case CMD_ENDFOOTNOTE:
                        if (closeCommand(cmd)) {
                            leavePara();
                            append(Atom::FootnoteRight);
                            paraState = InMultiLineParagraph; // ###
                        }
                        break;
                    case CMD_ENDIF:
                        if (preprocessorSkipping.count() > 0) {
                            if (preprocessorSkipping.pop())
                                --numPreprocessorSkipping;
                            (void)getRestOfLine(); // ### should ensure that it's empty
                            if (numPreprocessorSkipping)
                                skipToNextPreprocessorCommand();
                        }
                        else {
                            location().warning(tr("Unexpected '\\%1'").arg(cmdName(CMD_ENDIF)));
                        }
                        break;
                    case CMD_ENDLEGALESE:
                        if (closeCommand(cmd)) {
                            leavePara();
                            append(Atom::LegaleseRight);
                        }
                        break;
                    case CMD_ENDLINK:
                        if (closeCommand(cmd)) {
                            if (priv->text.lastAtom()->type() == Atom::String
                                    && priv->text.lastAtom()->string().endsWith(" "))
                                priv->text.lastAtom()->chopString();
                            append(Atom::FormattingRight, ATOM_FORMATTING_LINK);
                        }
                        break;
                    case CMD_ENDLIST:
                        if (closeCommand(cmd)) {
                            leavePara();
                            if (openedLists.top().isStarted()) {
                                append(Atom::ListItemRight,
                                        openedLists.top().styleString());
                                append(Atom::ListRight,
                                        openedLists.top().styleString());
                            }
                            openedLists.pop();
                        }
                        break;
                    case CMD_ENDOMIT:
                        closeCommand(cmd);
                        break;
                    case CMD_ENDPART:
                        endSection(Doc::Part, cmd);
                        break;
                    case CMD_ENDQUOTATION:
                        if (closeCommand(cmd)) {
                            leavePara();
                            append(Atom::QuotationRight);
                        }
                        break;
                    case CMD_ENDRAW:
                        location().warning(tr("Unexpected '\\%1'").arg(cmdName(CMD_ENDRAW)));
                        break;
                    case CMD_ENDSECTION1:
                        endSection(Doc::Section1, cmd);
                        break;
                    case CMD_ENDSECTION2:
                        endSection(Doc::Section2, cmd);
                        break;
                    case CMD_ENDSECTION3:
                        endSection(Doc::Section3, cmd);
                        break;
                    case CMD_ENDSECTION4:
                        endSection(Doc::Section4, cmd);
                        break;
                    case CMD_ENDSIDEBAR:
                        if (closeCommand(cmd)) {
                            leavePara();
                            append(Atom::SidebarRight);
                        }
                        break;
                    case CMD_ENDTABLE:
                        if (closeCommand(cmd)) {
                            leaveTableRow();
                            append(Atom::TableRight);
                        }
                        break;
                    case CMD_EXPIRE:
                        checkExpiry(getArgument());
                        break;
                    case CMD_FOOTNOTE:
                        if (openCommand(cmd)) {
                            enterPara();
                            append(Atom::FootnoteLeft);
                            paraState = OutsideParagraph; // ###
                        }
                        break;
                    case CMD_ANNOTATEDLIST:
                        append(Atom::AnnotatedList, getArgument());
                        break;
                    case CMD_SINCELIST:
                        append(Atom::SinceList, getRestOfLine().simplified());
                        break;
                    case CMD_GENERATELIST:
                        append(Atom::GeneratedList, getArgument());
                        break;
                    case CMD_GRANULARITY:
                        priv->constructExtra();
                        priv->extra->granularity = getSectioningUnit();
                        break;
                    case CMD_HEADER:
                        if (openedCommands.top() == CMD_TABLE) {
                            leaveTableRow();
                            append(Atom::TableHeaderLeft);
                            inTableHeader = true;
                        }
                        else {
                            if (openedCommands.contains(CMD_TABLE)) {
                                location().warning(tr("Cannot use '\\%1' within '\\%2'")
                                                   .arg(cmdName(CMD_HEADER))
                                                   .arg(cmdName(openedCommands.top())));
                            }
                            else {
                                location().warning(tr("Cannot use '\\%1' outside of '\\%2'")
                                                   .arg(cmdName(CMD_HEADER))
                                                   .arg(cmdName(CMD_TABLE)));
                            }
                        }
                        break;
                    case CMD_I:
                        startFormat(ATOM_FORMATTING_ITALIC, cmd);
                        break;
                    case CMD_IF:
                        preprocessorSkipping.push(!Tokenizer::isTrue(getRestOfLine()));
                        if (preprocessorSkipping.top())
                            ++numPreprocessorSkipping;
                        if (numPreprocessorSkipping)
                            skipToNextPreprocessorCommand();
                        break;
                    case CMD_IMAGE:
                        leaveValueList();
                        append(Atom::Image, getArgument());
                        append(Atom::ImageText, getRestOfLine());
                        break;
                    case CMD_INCLUDE:
                        {
                            QString fileName = getArgument();
                            QString identifier = getRestOfLine();
                            include(fileName, identifier);
                        }
                        break;
                    case CMD_INLINEIMAGE:
                        enterPara();
                        append(Atom::InlineImage, getArgument());
                        append(Atom::ImageText, getRestOfLine());
                        append(Atom::String, " ");
                        break;
                    case CMD_INDEX:
                        if (paraState == OutsideParagraph) {
                            enterPara();
                            indexStartedPara = true;
                        }
                        else {
                            const Atom *last = priv->text.lastAtom();
                            if (indexStartedPara &&
                                 (last->type() != Atom::FormattingRight ||
                                  last->string() != ATOM_FORMATTING_INDEX))
                                indexStartedPara = false;
                        }
                        startFormat(ATOM_FORMATTING_INDEX, cmd);
                        break;
                    case CMD_KEYWORD:
                        insertTarget(getRestOfLine(),true);
                        break;
                    case CMD_L:
                        enterPara();
                        if (isLeftBraceAhead()) {
                            p1 = getArgument();
                            append(Atom::Link, p1);
                            if (isLeftBraceAhead()) {
                                currentLinkAtom = priv->text.lastAtom();
                                startFormat(ATOM_FORMATTING_LINK, cmd);
                            }
                            else {
                                append(Atom::FormattingLeft, ATOM_FORMATTING_LINK);
                                append(Atom::String, cleanLink(p1));
                                append(Atom::FormattingRight, ATOM_FORMATTING_LINK);
                            }
                        }
                        else {
                            p1 = getArgument();
                            append(Atom::Link, p1);
                            append(Atom::FormattingLeft, ATOM_FORMATTING_LINK);
                            append(Atom::String, cleanLink(p1));
                            append(Atom::FormattingRight, ATOM_FORMATTING_LINK);
                        }
                        break;
                    case CMD_LEGALESE:
                        leavePara();
                        if (openCommand(cmd))
                            append(Atom::LegaleseLeft);
                        docPrivate->hasLegalese = true;
                        break;
                    case CMD_LINK:
                        if (openCommand(cmd)) {
                            enterPara();
                            p1 = getArgument();
                            append(Atom::Link, p1);
                            append(Atom::FormattingLeft, ATOM_FORMATTING_LINK);
                            skipSpacesOrOneEndl();
                        }
                        break;
                    case CMD_LIST:
                        if (openCommand(cmd)) {
                            leavePara();
                            openedLists.push(OpenedList(location(),
                                                         getOptionalArgument()));
                        }
                        break;
                    case CMD_META:
                        priv->constructExtra();
                        p1 = getArgument();
                        priv->extra->metaMap.insert(p1, getArgument());
                        break;
                    case CMD_NEWCODE:
                        location().warning(tr("Unexpected '\\%1'").arg(cmdName(CMD_NEWCODE)));
                        break;
                    case CMD_O:
                        leavePara();
                        if (openedCommands.top() == CMD_LIST) {
                            if (openedLists.top().isStarted()) {
                                append(Atom::ListItemRight,
                                        openedLists.top().styleString());
                            }
                            else {
                                append(Atom::ListLeft,
                                        openedLists.top().styleString());
                            }
                            openedLists.top().next();
                            append(Atom::ListItemNumber,
                                    openedLists.top().numberString());
                            append(Atom::ListItemLeft,
                                    openedLists.top().styleString());
                            enterPara();
                        }
                        else if (openedCommands.top() == CMD_TABLE) {
                            p1 = "1,1";
                            if (isLeftBraceAhead()) {
                                p1 = getArgument();
                                if (isLeftBraceAhead()) {
                                    p2 = getArgument();
                                }
                            }

                            if (!inTableHeader && !inTableRow) {
                                location().warning(tr("Missing '\\%1' or '\\%1' before '\\%3'")
                                                   .arg(cmdName(CMD_HEADER))
                                                   .arg(cmdName(CMD_ROW))
                                                   .arg(cmdName(CMD_O)));
                                append(Atom::TableRowLeft);
                                inTableRow = true;
                            }
                            else if (inTableItem) {
                                append(Atom::TableItemRight);
                                inTableItem = false;
                            }

                            append(Atom::TableItemLeft, p1, p2);
                            inTableItem = true;
                        }
                        else {
                            location().warning(tr("Command '\\%1' outside of '\\%2' and '\\%3'")
                                               .arg(cmdName(cmd))
                                               .arg(cmdName(CMD_LIST))
                                               .arg(cmdName(CMD_TABLE)));
                        }
                        break;
                    case CMD_OLDCODE:
                        leavePara();
                        append(Atom::CodeOld, getCode(CMD_OLDCODE, marker));
                        append(Atom::CodeNew, getCode(CMD_NEWCODE, marker));
                        break;
                    case CMD_OMIT:
                        getUntilEnd(cmd);
                        break;
                    case CMD_OMITVALUE:
                        p1 = getArgument();
                        if (!priv->enumItemList.contains(p1))
                            priv->enumItemList.append(p1);
                        if (!priv->omitEnumItemList.contains(p1))
                            priv->omitEnumItemList.append(p1);
                        break;
                    case CMD_PART:
                        startSection(Doc::Part, cmd);
                        break;
                    case CMD_PRINTLINE:
                        leavePara();
                        if (!quoting)
                            appendToCode(quoter.quoteLine(location(), cmdStr,
                                                           getRestOfLine()));
                        else {
                            append(Atom::CodeQuoteCommand, cmdStr);
                            append(Atom::CodeQuoteArgument, getRestOfLine());
                        }
                        break;
                    case CMD_PRINTTO:
                        leavePara();
                        if (!quoting)
                            appendToCode(quoter.quoteTo(location(), cmdStr,
                                          getRestOfLine()));
                        else {
                            append(Atom::CodeQuoteCommand, cmdStr);
                            append(Atom::CodeQuoteArgument, getRestOfLine());
                        }
                        break;
                    case CMD_PRINTUNTIL:
                        leavePara();
                        if (!quoting)
                            appendToCode(quoter.quoteUntil(location(), cmdStr,
                                          getRestOfLine()));
                        else {
                            append(Atom::CodeQuoteCommand, cmdStr);
                            append(Atom::CodeQuoteArgument, getRestOfLine());
                        }
                        break;
                    case CMD_QUOTATION:
                        if (openCommand(cmd)) {
                            leavePara();
                            append(Atom::QuotationLeft);
                        }
                        break;
                    case CMD_QUOTEFILE:
                    {
                        leavePara();
                        QString fileName = getArgument();
                        Doc::quoteFromFile(location(), quoter, fileName);
                        if (!quoting) {
                            append(Atom::Code,
                                    quoter.quoteTo(location(), cmdStr, ""));
                            quoter.reset();
                        }
                        else {
                            append(Atom::CodeQuoteCommand, cmdStr);
                            append(Atom::CodeQuoteArgument, fileName);
                        }
                        break;
                    }
                    case CMD_QUOTEFROMFILE:
                        leavePara();
                        if (!quoting)
                            quoteFromFile();
                        else {
                            append(Atom::CodeQuoteCommand, cmdStr);
                            append(Atom::CodeQuoteArgument, getArgument());
                        }
                        break;
                    case CMD_QUOTEFUNCTION:
                        leavePara();
                        marker = quoteFromFile();
                        p1 = getRestOfLine();
                        if (!quoting) {
                            quoter.quoteTo(location(), cmdStr,
                                            slashed(marker->functionBeginRegExp(p1)));
                            append(Atom::Code,
                                    quoter.quoteUntil(location(), cmdStr,
                                            slashed(marker->functionEndRegExp(p1))));
                            quoter.reset();
                        }
                        else {
                            append(Atom::CodeQuoteCommand, cmdStr);
                            append(Atom::CodeQuoteArgument, slashed(marker->functionEndRegExp(p1)));
                        }
                        break;
                    case CMD_RAW:
                        leavePara();
                        p1 = getRestOfLine();
                        if (p1.isEmpty())
                            location().warning(tr("Missing format name after '\\%1")
                                               .arg(cmdName(CMD_RAW)));
                        append(Atom::FormatIf, p1);
                        append(Atom::RawString, untabifyEtc(getUntilEnd(cmd)));
                        append(Atom::FormatElse);
                        append(Atom::FormatEndif);
                        break;
                    case CMD_ROW:
                        if (openedCommands.top() == CMD_TABLE) {
                            p1.clear();
                            if (isLeftBraceAhead())
                                p1 = getArgument(true);
                            leaveTableRow();
                            append(Atom::TableRowLeft,p1);
                            inTableRow = true;
                        }
                        else {
                            if (openedCommands.contains(CMD_TABLE)) {
                                location().warning(tr("Cannot use '\\%1' within '\\%2'")
                                                   .arg(cmdName(CMD_ROW))
                                                   .arg(cmdName(openedCommands.top())));
                            }
                            else {
                                location().warning(tr("Cannot use '\\%1' outside of '\\%2'")
                                                   .arg(cmdName(CMD_ROW))
                                                   .arg(cmdName(CMD_TABLE)));
                            }
                        }
                        break;
                    case CMD_SA:
                        parseAlso();
                        break;
                    case CMD_SECTION1:
                        startSection(Doc::Section1, cmd);
                        break;
                    case CMD_SECTION2:
                        startSection(Doc::Section2, cmd);
                        break;
                    case CMD_SECTION3:
                        startSection(Doc::Section3, cmd);
                        break;
                    case CMD_SECTION4:
                        startSection(Doc::Section4, cmd);
                        break;
                    case CMD_SIDEBAR:
                        if (openCommand(cmd)) {
                            leavePara();
                            append(Atom::SidebarLeft);
                        }
                        break;
                    case CMD_SKIPLINE:
                        leavePara();
                        if (!quoting)
                            quoter.quoteLine(location(),
                                             cmdStr,
                                             getRestOfLine());
                        else {
                            append(Atom::CodeQuoteCommand, cmdStr);
                            append(Atom::CodeQuoteArgument, getRestOfLine());
                        }
                        break;
                    case CMD_SKIPTO:
                        leavePara();
                        if (!quoting)
                            quoter.quoteTo(location(),
                                           cmdStr,
                                           getRestOfLine());
                        else {
                            append(Atom::CodeQuoteCommand, cmdStr);
                            append(Atom::CodeQuoteArgument, getRestOfLine());
                        }
                        break;
                    case CMD_SKIPUNTIL:
                        leavePara();
                        if (!quoting)
                            quoter.quoteUntil(location(),
                                              cmdStr,
                                              getRestOfLine());
                        else {
                            append(Atom::CodeQuoteCommand, cmdStr);
                            append(Atom::CodeQuoteArgument, getRestOfLine());
                        }
                        break;
                    case CMD_SPAN:
                        p1 = ATOM_FORMATTING_SPAN + getArgument(true);
                        startFormat(p1, cmd);
                        break;
                    case CMD_SNIPPET:
                        leavePara();
                        {
                            QString snippet = getArgument();
                            QString identifier = getRestOfLine();
                            if (quoting) {
                                append(Atom::SnippetCommand, cmdStr);
                                append(Atom::SnippetLocation, snippet);
                                append(Atom::SnippetIdentifier, identifier);
                            }
                            else {
                                marker = Doc::quoteFromFile(location(),quoter,snippet);
                                appendToCode(quoter.quoteSnippet(location(), identifier), marker->atomType());
                            }
                        }
                        break;
                    case CMD_SUB:
                        startFormat(ATOM_FORMATTING_SUBSCRIPT, cmd);
                        break;
                    case CMD_SUP:
                        startFormat(ATOM_FORMATTING_SUPERSCRIPT, cmd);
                        break;
                    case CMD_TABLE:
                        p1 = getRestOfLine();
                        if (openCommand(cmd)) {
                            leavePara();
                            append(Atom::TableLeft, p1);
                            inTableHeader = false;
                            inTableRow = false;
                            inTableItem = false;
                        }
                        break;
                    case CMD_TABLEOFCONTENTS:
                        p1 = "1";
                        if (isLeftBraceAhead())
                            p1 = getArgument();
                        p1 += ",";
                        p1 += QString::number((int)getSectioningUnit());
                        append(Atom::TableOfContents, p1);
                        break;
                    case CMD_TARGET:
                        insertTarget(getRestOfLine(),false);
                        break;
                    case CMD_TT:
                        startFormat(ATOM_FORMATTING_TELETYPE, cmd);
                        break;
                    case CMD_UNDERLINE:
                        startFormat(ATOM_FORMATTING_UNDERLINE, cmd);
                        break;
                    case CMD_UNICODE:
                        enterPara();
                        p1 = getArgument();
                        {
                            bool ok;
                            uint unicodeChar = p1.toUInt(&ok, 0);
                            if (!ok ||
                                (unicodeChar == 0x0000) ||
                                (unicodeChar > 0xFFFE)) {
                                location().warning(tr("Invalid Unicode character '%1' specified "
                                                      "with '%2'")
                                                   .arg(p1, cmdName(CMD_UNICODE)));
                            }
                            else {
                                append(Atom::String, QChar(unicodeChar));
                            }
                        }
                        break;
                    case CMD_VALUE:
                        leaveValue();
                        if (openedLists.top().style() == OpenedList::Value) {
                            p1 = getArgument();
                            if (!priv->enumItemList.contains(p1))
                                priv->enumItemList.append(p1);

                            openedLists.top().next();
                            append(Atom::ListTagLeft, ATOM_LIST_VALUE);
                            append(Atom::String, p1);
                            append(Atom::ListTagRight, ATOM_LIST_VALUE);
                            append(Atom::ListItemLeft, ATOM_LIST_VALUE);

                            skipSpacesOrOneEndl();
                            if (isBlankLine())
                                append(Atom::Nop);
                        }
                        else {
                            // ### problems
                        }
                        break;
                    case CMD_WARNING:
                        leavePara();
                        enterPara();
                        append(Atom::FormattingLeft, ATOM_FORMATTING_BOLD);
                        app…