/src/wp/csharp-sqlite-src/tokenize_c.cs

using System;
using System.Diagnostics;
using System.Text;

namespace Community.CsharpSqlite
{
  public partial class Sqlite3
  {
    /*
    ** 2001 September 15
    **
    ** The author disclaims copyright to this source code.  In place of
    ** a legal notice, here is a blessing:
    **
    **    May you do good and not evil.
    **    May you find forgiveness for yourself and forgive others.
    **    May you share freely, never taking more than you give.
    **
    *************************************************************************
    ** An tokenizer for SQL
    **
    ** This file contains C code that splits an SQL input string up into
    ** individual tokens and sends those tokens one-by-one over to the
    ** parser for analysis.
    *************************************************************************
    **  Included in SQLite3 port to C#-SQLite;  2008 Noah B Hart
    **  C#-SQLite is an independent reimplementation of the SQLite software library
    **
    **  SQLITE_SOURCE_ID: 2011-06-23 19:49:22 4374b7e83ea0a3fbc3691f9c0c936272862f32f2
    **
    *************************************************************************
    */
    //#include "sqliteInt.h"
    //#include <stdlib.h>

    /*
    ** The charMap() macro maps alphabetic characters into their
    ** lower-case ASCII equivalent.  On ASCII machines, this is just
    ** an upper-to-lower case map.  On EBCDIC machines we also need
    ** to adjust the encoding.  Only alphabetic characters and underscores
    ** need to be translated.
    */
#if SQLITE_ASCII
    //# define charMap(X) sqlite3UpperToLower[(unsigned char)X]
#endif
//#if SQLITE_EBCDIC
//# define charMap(X) ebcdicToAscii[(unsigned char)X]
//const unsigned char ebcdicToAscii[] = {
///* 0   1   2   3   4   5   6   7   8   9   A   B   C   D   E   F */
//   0,  0,  0,  0,  0,  0,  0,  0,  0,  0,  0,  0,  0,  0,  0,  0,  /* 0x */
//   0,  0,  0,  0,  0,  0,  0,  0,  0,  0,  0,  0,  0,  0,  0,  0,  /* 1x */
//   0,  0,  0,  0,  0,  0,  0,  0,  0,  0,  0,  0,  0,  0,  0,  0,  /* 2x */
//   0,  0,  0,  0,  0,  0,  0,  0,  0,  0,  0,  0,  0,  0,  0,  0,  /* 3x */
//   0,  0,  0,  0,  0,  0,  0,  0,  0,  0,  0,  0,  0,  0,  0,  0,  /* 4x */
//   0,  0,  0,  0,  0,  0,  0,  0,  0,  0,  0,  0,  0,  0,  0,  0,  /* 5x */
//   0,  0,  0,  0,  0,  0,  0,  0,  0,  0,  0,  0,  0, 95,  0,  0,  /* 6x */
//   0,  0,  0,  0,  0,  0,  0,  0,  0,  0,  0,  0,  0,  0,  0,  0,  /* 7x */
//   0, 97, 98, 99,100,101,102,103,104,105,  0,  0,  0,  0,  0,  0,  /* 8x */
//   0,106,107,108,109,110,111,112,113,114,  0,  0,  0,  0,  0,  0,  /* 9x */
//   0,  0,115,116,117,118,119,120,121,122,  0,  0,  0,  0,  0,  0,  /* Ax */
//   0,  0,  0,  0,  0,  0,  0,  0,  0,  0,  0,  0,  0,  0,  0,  0,  /* Bx */
//   0, 97, 98, 99,100,101,102,103,104,105,  0,  0,  0,  0,  0,  0,  /* Cx */
//   0,106,107,108,109,110,111,112,113,114,  0,  0,  0,  0,  0,  0,  /* Dx */
//   0,  0,115,116,117,118,119,120,121,122,  0,  0,  0,  0,  0,  0,  /* Ex */
//   0,  0,  0,  0,  0,  0,  0,  0,  0,  0,  0,  0,  0,  0,  0,  0,  /* Fx */
//};
//#endif

    /*
** The sqlite3KeywordCode function looks up an identifier to determine if
** it is a keyword.  If it is a keyword, the token code of that keyword is
** returned.  If the input is not a keyword, TK_ID is returned.
**
** The implementation of this routine was generated by a program,
** mkkeywordhash.h, located in the tool subdirectory of the distribution.
** The output of the mkkeywordhash.c program is written into a file
** named keywordhash.h and then included into this source file by
** the #include below.
*/
    //#include "keywordhash.h"


    /*
    ** If X is a character that can be used in an identifier then
    ** IdChar(X) will be true.  Otherwise it is false.
    **
    ** For ASCII, any character with the high-order bit set is
    ** allowed in an identifier.  For 7-bit characters,
    ** sqlite3IsIdChar[X] must be 1.
    **
    ** For EBCDIC, the rules are more complex but have the same
    ** end result.
    **
    ** Ticket #1066.  the SQL standard does not allow '$' in the
    ** middle of identfiers.  But many SQL implementations do.
    ** SQLite will allow '$' in identifiers for compatibility.
    ** But the feature is undocumented.
    */
#if SQLITE_ASCII
    //#define IdChar(C)  ((sqlite3CtypeMap[(unsigned char)C]&0x46)!=0)
#endif
//#if SQLITE_EBCDIC
//const char sqlite3IsEbcdicIdChar[] = {
///* x0 x1 x2 x3 x4 x5 x6 x7 x8 x9 xA xB xC xD xE xF */
//    0, 0, 1, 1, 1, 1, 1, 1, 1, 1, 0, 0, 0, 0, 0, 0,  /* 4x */
//    0, 1, 1, 1, 1, 1, 1, 1, 1, 1, 0, 1, 0, 0, 0, 0,  /* 5x */
//    0, 0, 1, 1, 1, 1, 1, 1, 1, 1, 0, 0, 0, 1, 0, 0,  /* 6x */
//    0, 1, 1, 1, 1, 1, 1, 1, 1, 0, 0, 0, 0, 0, 0, 0,  /* 7x */
//    0, 1, 1, 1, 1, 1, 1, 1, 1, 1, 0, 0, 1, 1, 1, 0,  /* 8x */
//    0, 1, 1, 1, 1, 1, 1, 1, 1, 1, 0, 0, 1, 0, 1, 0,  /* 9x */
//    1, 0, 1, 1, 1, 1, 1, 1, 1, 1, 1, 0, 1, 1, 1, 0,  /* Ax */
//    0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0,  /* Bx */
//    0, 1, 1, 1, 1, 1, 1, 1, 1, 1, 0, 1, 1, 1, 1, 1,  /* Cx */
//    0, 1, 1, 1, 1, 1, 1, 1, 1, 1, 0, 1, 1, 1, 1, 1,  /* Dx */
//    0, 0, 1, 1, 1, 1, 1, 1, 1, 1, 0, 1, 1, 1, 1, 1,  /* Ex */
//    1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 0, 1, 1, 1, 1, 0,  /* Fx */
//};
//#define IdChar(C)  (((c=C)>=0x42 && sqlite3IsEbcdicIdChar[c-0x40]))
//#endif


    /*
** Return the length of the token that begins at z[iOffset + 0].
** Store the token type in *tokenType before returning.
*/
    static int sqlite3GetToken( string z, int iOffset, ref int tokenType )
    {
      int i;
      byte c = 0;
      switch ( z[iOffset + 0] )
      {
        case ' ':
        case '\t':
        case '\n':
        case '\f':
        case '\r':
          {
            testcase( z[iOffset + 0] == ' ' );
            testcase( z[iOffset + 0] == '\t' );
            testcase( z[iOffset + 0] == '\n' );
            testcase( z[iOffset + 0] == '\f' );
            testcase( z[iOffset + 0] == '\r' );
            for ( i = 1; z.Length > iOffset + i && sqlite3Isspace( z[iOffset + i] ); i++ )
            {
            }
            tokenType = TK_SPACE;
            return i;
          }
        case '-':
          {
            if ( z.Length > iOffset + 1 && z[iOffset + 1] == '-' )
            {
              /* IMP: R-15891-05542 -- syntax diagram for comments */
              for ( i = 2; z.Length > iOffset + i && ( c = (byte)z[iOffset + i] ) != 0 && c != '\n'; i++ )
              {
              }
              tokenType = TK_SPACE;   /* IMP: R-22934-25134 */
              return i;
            }
            tokenType = TK_MINUS;
            return 1;
          }
        case '(':
          {
            tokenType = TK_LP;
            return 1;
          }
        case ')':
          {
            tokenType = TK_RP;
            return 1;
          }
        case ';':
          {
            tokenType = TK_SEMI;
            return 1;
          }
        case '+':
          {
            tokenType = TK_PLUS;
            return 1;
          }
        case '*':
          {
            tokenType = TK_STAR;
            return 1;
          }
        case '/':
          {
            if ( iOffset + 2 >= z.Length || z[iOffset + 1] != '*' )
            {
              tokenType = TK_SLASH;
              return 1;
            }
            /* IMP: R-15891-05542 -- syntax diagram for comments */
            for ( i = 3, c = (byte)z[iOffset + 2]; iOffset + i < z.Length && ( c != '*' || ( z[iOffset + i] != '/' ) && ( c != 0 ) ); i++ )
            {
              c = (byte)z[iOffset + i];
            }
            if ( iOffset + i == z.Length )
              c = 0;
            if ( c != 0 )
              i++;
            tokenType = TK_SPACE; /* IMP: R-22934-25134 */
            return i;
          }
        case '%':
          {
            tokenType = TK_REM;
            return 1;
          }
        case '=':
          {
            tokenType = TK_EQ;
            return 1 + ( z[iOffset + 1] == '=' ? 1 : 0 );
          }
        case '<':
          {
            if ( ( c = (byte)z[iOffset + 1] ) == '=' )
            {
              tokenType = TK_LE;
              return 2;
            }
            else if ( c == '>' )
            {
              tokenType = TK_NE;
              return 2;
            }
            else if ( c == '<' )
            {
              tokenType = TK_LSHIFT;
              return 2;
            }
            else
            {
              tokenType = TK_LT;
              return 1;
            }
          }
        case '>':
          {
            if ( z.Length > iOffset + 1 && ( c = (byte)z[iOffset + 1] ) == '=' )
            {
              tokenType = TK_GE;
              return 2;
            }
            else if ( c == '>' )
            {
              tokenType = TK_RSHIFT;
              return 2;
            }
            else
            {
              tokenType = TK_GT;
              return 1;
            }
          }
        case '!':
          {
            if ( z[iOffset + 1] != '=' )
            {
              tokenType = TK_ILLEGAL;
              return 2;
            }
            else
            {
              tokenType = TK_NE;
              return 2;
            }
          }
        case '|':
          {
            if ( z[iOffset + 1] != '|' )
            {
              tokenType = TK_BITOR;
              return 1;
            }
            else
            {
              tokenType = TK_CONCAT;
              return 2;
            }
          }
        case ',':
          {
            tokenType = TK_COMMA;
            return 1;
          }
        case '&':
          {
            tokenType = TK_BITAND;
            return 1;
          }
        case '~':
          {
            tokenType = TK_BITNOT;
            return 1;
          }
        case '`':
        case '\'':
        case '"':
          {
            int delim = z[iOffset + 0];
            testcase( delim == '`' );
            testcase( delim == '\'' );
            testcase( delim == '"' );
            for ( i = 1; ( iOffset + i ) < z.Length && ( c = (byte)z[iOffset + i] ) != 0; i++ )
            {
              if ( c == delim )
              {
                if ( z.Length > iOffset + i + 1 && z[iOffset + i + 1] == delim )
                {
                  i++;
                }
                else
                {
                  break;
                }
              }
            }
            if ( ( iOffset + i == z.Length && c != delim ) || z[iOffset + i] != delim )
            {
              tokenType = TK_ILLEGAL;
              return i + 1;
            }
            if ( c == '\'' )
            {
              tokenType = TK_STRING;
              return i + 1;
            }
            else if ( c != 0 )
            {
              tokenType = TK_ID;
              return i + 1;
            }
            else
            {
              tokenType = TK_ILLEGAL;
              return i;
            }
          }
        case '.':
          {
#if !SQLITE_OMIT_FLOATING_POINT
            if ( !sqlite3Isdigit( z[iOffset + 1] ) )
#endif
            {
              tokenType = TK_DOT;
              return 1;
            }
            /* If the next character is a digit, this is a floating point
            ** number that begins with ".".  Fall thru into the next case */
            goto case '0';
          }
        case '0':
        case '1':
        case '2':
        case '3':
        case '4':
        case '5':
        case '6':
        case '7':
        case '8':
        case '9':
          {
            testcase( z[iOffset] == '0' );
            testcase( z[iOffset] == '1' );
            testcase( z[iOffset] == '2' );
            testcase( z[iOffset] == '3' );
            testcase( z[iOffset] == '4' );
            testcase( z[iOffset] == '5' );
            testcase( z[iOffset] == '6' );
            testcase( z[iOffset] == '7' );
            testcase( z[iOffset] == '8' );
            testcase( z[iOffset] == '9' );
            tokenType = TK_INTEGER;
            for ( i = 0; z.Length > iOffset + i && sqlite3Isdigit( z[iOffset + i] ); i++ )
            {
            }
#if !SQLITE_OMIT_FLOATING_POINT
            if ( z.Length > iOffset + i && z[iOffset + i] == '.' )
            {
              i++;
              while ( z.Length > iOffset + i && sqlite3Isdigit( z[iOffset + i] ) )
              {
                i++;
              }
              tokenType = TK_FLOAT;
            }
            if ( z.Length > iOffset + i + 1 && ( z[iOffset + i] == 'e' || z[iOffset + i] == 'E' ) &&
            ( sqlite3Isdigit( z[iOffset + i + 1] )
            || z.Length > iOffset + i + 2 && ( ( z[iOffset + i + 1] == '+' || z[iOffset + i + 1] == '-' ) && sqlite3Isdigit( z[iOffset + i + 2] ) )
            )
            )
            {
              i += 2;
              while ( z.Length > iOffset + i && sqlite3Isdigit( z[iOffset + i] ) )
              {
                i++;
              }
              tokenType = TK_FLOAT;
            }
#endif
            while ( iOffset + i < z.Length && IdChar( (byte)z[iOffset + i] ) )
            {
              tokenType = TK_ILLEGAL;
              i++;
            }
            return i;
          }

        case '[':
          {
            for ( i = 1, c = (byte)z[iOffset + 0]; c != ']' && ( iOffset + i ) < z.Length && ( c = (byte)z[iOffset + i] ) != 0; i++ )
            {
            }
            tokenType = c == ']' ? TK_ID : TK_ILLEGAL;
            return i;
          }
        case '?':
          {
            tokenType = TK_VARIABLE;
            for ( i = 1; z.Length > iOffset + i && sqlite3Isdigit( z[iOffset + i] ); i++ )
            {
            }
            return i;
          }
        case '#':
          {
            for ( i = 1; z.Length > iOffset + i && sqlite3Isdigit( z[iOffset + i] ); i++ )
            {
            }
            if ( i > 1 )
            {
              /* Parameters of the form #NNN (where NNN is a number) are used
              ** internally by sqlite3NestedParse.  */
              tokenType = TK_REGISTER;
              return i;
            }
            /* Fall through into the next case if the '#' is not followed by
            ** a digit. Try to match #AAAA where AAAA is a parameter name. */
            goto case ':';
          }
#if !SQLITE_OMIT_TCL_VARIABLE
        case '$':
#endif
        case '@':  /* For compatibility with MS SQL Server */
        case ':':
          {
            int n = 0;
            testcase( z[iOffset + 0] == '$' );
            testcase( z[iOffset + 0] == '@' );
            testcase( z[iOffset + 0] == ':' );
            tokenType = TK_VARIABLE;
            for ( i = 1; z.Length > iOffset + i && ( c = (byte)z[iOffset + i] ) != 0; i++ )
            {
              if ( IdChar( c ) )
              {
                n++;
#if !SQLITE_OMIT_TCL_VARIABLE
              }
              else if ( c == '(' && n > 0 )
              {
                do
                {
                  i++;
                } while ( ( iOffset + i ) < z.Length && ( c = (byte)z[iOffset + i] ) != 0 && !sqlite3Isspace( c ) && c != ')' );
                if ( c == ')' )
                {
                  i++;
                }
                else
                {
                  tokenType = TK_ILLEGAL;
                }
                break;
              }
              else if ( c == ':' && z[iOffset + i + 1] == ':' )
              {
                i++;
#endif
              }
              else
              {
                break;
              }
            }
            if ( n == 0 )
              tokenType = TK_ILLEGAL;
            return i;
          }
#if !SQLITE_OMIT_BLOB_LITERAL
        case 'x':
        case 'X':
          {
            testcase( z[iOffset + 0] == 'x' );
            testcase( z[iOffset + 0] == 'X' );
            if ( z.Length > iOffset + 1 && z[iOffset + 1] == '\'' )
            {
              tokenType = TK_BLOB;
                for ( i = 2; z.Length > iOffset + i && sqlite3Isxdigit( z[iOffset + i] ); i++ )
                {
                }
                if ( iOffset + i == z.Length || z[iOffset + i] != '\'' || i % 2 != 0 )
              {
                tokenType = TK_ILLEGAL;
                while ( z.Length > iOffset + i && z[iOffset + i] != '\'' )
                {
                  i++;
                }
              }
              if ( z.Length > iOffset + i )
                i++;
              return i;
            }
            goto default;
            /* Otherwise fall through to the next case */
          }
#endif
        default:
          {
            if ( !IdChar( (byte)z[iOffset] ) )
            {
              break;
            }
            for ( i = 1; i < z.Length - iOffset && IdChar( (byte)z[iOffset + i] ); i++ )
            {
            }
            tokenType = keywordCode( z, iOffset, i );
            return i;
          }
      }
      tokenType = TK_ILLEGAL;
      return 1;
    }

    /*
    ** Run the parser on the given SQL string.  The parser structure is
    ** passed in.  An SQLITE_ status code is returned.  If an error occurs
    ** then an and attempt is made to write an error message into
    ** memory obtained from sqlite3_malloc() and to make pzErrMsg point to that
    ** error message.
    */
    static int sqlite3RunParser( Parse pParse, string zSql, ref string pzErrMsg )
    {
      int nErr = 0;                   /* Number of errors encountered */
      int i;                          /* Loop counter */
      yyParser pEngine;               /* The LEMON-generated LALR(1) parser */
      int tokenType = 0;              /* type of the next token */
      int lastTokenParsed = -1;       /* type of the previous token */
      byte enableLookaside;           /* Saved value of db->lookaside.bEnabled */
      sqlite3 db = pParse.db;         /* The database connection */
      int mxSqlLen;                   /* Max length of an SQL string */


      mxSqlLen = db.aLimit[SQLITE_LIMIT_SQL_LENGTH];
      if ( db.activeVdbeCnt == 0 )
      {
        db.u1.isInterrupted = false;
      }
      pParse.rc = SQLITE_OK;
      pParse.zTail = new StringBuilder( zSql );
      i = 0;
      Debug.Assert( pzErrMsg != null );
      pEngine = sqlite3ParserAlloc();//sqlite3ParserAlloc((void*(*)(size_t))sqlite3Malloc);
      //if ( pEngine == null )
      //{
      //  db.mallocFailed = 1;
      //  return SQLITE_NOMEM;
      //}
      Debug.Assert( pParse.pNewTable == null );
      Debug.Assert( pParse.pNewTrigger == null );
      Debug.Assert( pParse.nVar == 0 );
      Debug.Assert( pParse.nzVar == 0 );
      Debug.Assert( pParse.azVar == null );
      enableLookaside = db.lookaside.bEnabled;
      if ( db.lookaside.pStart != 0 )
        db.lookaside.bEnabled = 1;
      while ( /*  0 == db.mallocFailed && */  i < zSql.Length )
      {
        Debug.Assert( i >= 0 );
        //pParse->sLastToken.z = &zSql[i];
        pParse.sLastToken.n = sqlite3GetToken( zSql, i, ref tokenType );
        pParse.sLastToken.z = zSql.Substring( i );
        i += pParse.sLastToken.n;
        if ( i > mxSqlLen )
        {
          pParse.rc = SQLITE_TOOBIG;
          break;
        }
        switch ( tokenType )
        {
          case TK_SPACE:
            {
              if ( db.u1.isInterrupted )
              {
                sqlite3ErrorMsg( pParse, "interrupt" );
                pParse.rc = SQLITE_INTERRUPT;
                goto abort_parse;
              }
              break;
            }
          case TK_ILLEGAL:
            {
              sqlite3DbFree( db, ref pzErrMsg );
              pzErrMsg = sqlite3MPrintf( db, "unrecognized token: \"%T\"",
                (object)pParse.sLastToken );
              nErr++;
              goto abort_parse;
            }
          case TK_SEMI:
            {
              //pParse.zTail = new StringBuilder(zSql.Substring( i,zSql.Length-i ));
              /* Fall thru into the default case */
              goto default;
            }
          default:
            {
              sqlite3Parser( pEngine, tokenType, pParse.sLastToken, pParse );
              lastTokenParsed = tokenType;
              if ( pParse.rc != SQLITE_OK )
              {
                goto abort_parse;
              }
              break;
            }
        }
      }
abort_parse:
      pParse.zTail = new StringBuilder( zSql.Length <= i ? "" : zSql.Substring( i, zSql.Length - i ) );
      if ( zSql.Length >= i && nErr == 0 && pParse.rc == SQLITE_OK )
      {
        if ( lastTokenParsed != TK_SEMI )
        {
          sqlite3Parser( pEngine, TK_SEMI, pParse.sLastToken, pParse );
        }
        sqlite3Parser( pEngine, 0, pParse.sLastToken, pParse );
      }
#if YYTRACKMAXSTACKDEPTH
sqlite3StatusSet(SQLITE_STATUS_PARSER_STACK,
sqlite3ParserStackPeak(pEngine)
);
#endif //* YYDEBUG */
      sqlite3ParserFree( pEngine, null );//sqlite3_free );
      db.lookaside.bEnabled = enableLookaside;
      //if ( db.mallocFailed != 0 )
      //{
      //  pParse.rc = SQLITE_NOMEM;
      //}
      if ( pParse.rc != SQLITE_OK && pParse.rc != SQLITE_DONE && pParse.zErrMsg == "" )
      {
        sqlite3SetString( ref pParse.zErrMsg, db, sqlite3ErrStr( pParse.rc ) );
      }
      //assert( pzErrMsg!=0 );
      if ( pParse.zErrMsg != null )
      {
        pzErrMsg = pParse.zErrMsg;
        sqlite3_log( pParse.rc, "%s", pzErrMsg );
        pParse.zErrMsg = "";
        nErr++;
      }
      if ( pParse.pVdbe != null && pParse.nErr > 0 && pParse.nested == 0 )
      {
        sqlite3VdbeDelete( ref pParse.pVdbe );
        pParse.pVdbe = null;
      }
#if NO_SQLITE_OMIT_SHARED_CACHE
//#if !SQLITE_OMIT_SHARED_CACHE
if ( pParse.nested == 0 )
{
sqlite3DbFree( db, ref pParse.aTableLock );
pParse.aTableLock = null;
pParse.nTableLock = 0;
}
#endif
#if !SQLITE_OMIT_VIRTUALTABLE
      pParse.apVtabLock = null;//sqlite3_free( pParse.apVtabLock );
#endif
      if ( !IN_DECLARE_VTAB(pParse) )
      {
        /* If the pParse.declareVtab flag is set, do not delete any table
        ** structure built up in pParse.pNewTable. The calling code (see vtab.c)
        ** will take responsibility for freeing the Table structure.
        */
        sqlite3DeleteTable( db, ref pParse.pNewTable );
      }

#if !SQLITE_OMIT_TRIGGER
      sqlite3DeleteTrigger( db, ref pParse.pNewTrigger );
#endif
      //for ( i = pParse.nzVar - 1; i >= 0; i-- )
      //  sqlite3DbFree( db, pParse.azVar[i] );
      sqlite3DbFree( db, ref pParse.azVar );
      sqlite3DbFree( db, ref pParse.aAlias );
      while ( pParse.pAinc != null )
      {
        AutoincInfo p = pParse.pAinc;
        pParse.pAinc = p.pNext;
        sqlite3DbFree( db, ref p );
      }
      while ( pParse.pZombieTab != null )
      {
        Table p = pParse.pZombieTab;
        pParse.pZombieTab = p.pNextZombie;
        sqlite3DeleteTable( db, ref p );
      }
      if ( nErr > 0 && pParse.rc == SQLITE_OK )
      {
        pParse.rc = SQLITE_ERROR;
      }
      return nErr;
    }
  }
}