/ecl/hql/hqlgram.y

Large files files are truncated, but you can click here to view the full file

//
//    HPCC SYSTEMS software Copyright (C) 2012 HPCC Systems.
//
//    Licensed under the Apache License, Version 2.0 (the "License");
//    you may not use this file except in compliance with the License.
//    You may obtain a copy of the License at
//
//       http://www.apache.org/licenses/LICENSE-2.0
//
//    Unless required by applicable law or agreed to in writing, software
//    distributed under the License is distributed on an "AS IS" BASIS,
//    WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
//    See the License for the specific language governing permissions and
//    limitations under the License.
//############################################################################## */

//Either api.pure of c++ skeleton could be used, not both (which causes an error).
//Note the c++ generation still generates a separate class for the raw processing from the HqlGram class, so whichever is
//used the productions need to use parser->... to access the context
%define api.pure
//%error-verbose
%lex-param {HqlGram* parser}
%lex-param {int * yyssp}
%parse-param {HqlGram* parser}
%name-prefix "eclyy"
//
%destructor {$$.release();} <>
//Could override destructors for all tokens e.g.,
//%destructor {} ABS
//but warnings still come out, and improvement in code is marginal.
//Only defining destructors for those productions that need them would solve it, but be open to missing items.
//Adding a comment to reference unused parameters also fails to solve it because it ignores references in comments (and a bit ugly)
//fixing bison to ignore destructor {} for need use is another alternative - but would take a long time to feed into a public build.
%{
#include "platform.h"
#include <stdio.h>
#include <stdlib.h>
#include "hql.hpp"
#include "jlib.hpp"
#include "jmisc.hpp"
#include "jexcept.hpp"
#include "hqlerrors.hpp"

#include "hqlgram.hpp"
#include "hqlfold.hpp"
#include "hqlpmap.hpp"
#include "hqlutil.hpp"
#include "hqlattr.hpp"
#include "hqlmeta.hpp"

#define REDEF_MSG(name)     StringBuffer msg;           \
                            msg.append(w"Identifier '"); \
                            msg.append(name.queryExpr()->queryName()->str()); \
                            msg.append("' before := ");                       \
                            msg.append(" is already defined");                 \
                            parser->reportError(ERR_ID_REDEFINE, name, msg.str())

#define REDEF_ERROR(name)                    \
                        {                    \
                            REDEF_MSG(name); \
                        }

#define REDEF_ERROR1(name,e1)                \
                        {                    \
                            REDEF_MSG(name); \
                            e1.release();    \
                        }

#define REDEF_ERROR2(name,e1,e2)             \
                        {                    \
                            REDEF_MSG(name); \
                            e1.release();    \
                            e2.release();    \
                        }

#define REDEF_ERROR3(name,e1,e2,e3)          \
                        {                    \
                            REDEF_MSG(name); \
                            e1.release();    \
                            e2.release();    \
                            e3.release();    \
                        }

inline int eclyylex(attribute * yylval, HqlGram* parser, const short int * yyssp)
{
    return parser->yyLex(yylval, yyssp);
}



static void eclsyntaxerror(HqlGram * parser, const char * s, short yystate, int token);
#define eclyyerror(parser, s)       eclsyntaxerror(parser, s, yystate, yychar)
#define ignoreBisonWarning(x)
#define ignoreBisonWarnings2(x,y)
#define ignoreBisonWarnings3(x,y,z)

%}

//=========================================== tokens ====================================

%token
/* remember to add any new tokens to the error reporter and lexer too! */
/* If they clash with other #defines etc then use TOK_ as a prefix */

  ABS
  ACOS
  AFTER
  AGGREGATE
  ALIAS
  ALL
  ALLNODES
  AND
  ANY
  APPLY
  _ARRAY_
  AS
  ASCII
  ASIN
  TOK_ASSERT
  ASSTRING
  ATAN
  ATAN2
  ATMOST
  AVE
  BACKUP
  BEFORE
  BEST
  BETWEEN
  TOK_BITMAP
  BIG
  BLOB
  BNOT
  BUILD
  CARDINALITY
  CASE
  TOK_CATCH
  CHECKPOINT
  CHOOSE
  CHOOSEN
  CHOOSENALL
  CHOOSESETS
  CLUSTER
  CLUSTERSIZE
  COGROUP
  __COMMON__
  __COMPOUND__
  COMBINE
  COMPRESSED
  __COMPRESSED__
  TOK_CONST
  CORRELATION
  COS
  COSH
  COUNT
  COUNTER
  COVARIANCE
  CPPBODY
  CRC
  CRON
  CSV
  DATASET
  __DEBUG__
  DEDUP
  DEFAULT
  DEFINE
  DENORMALIZE
  DEPRECATED
  DESC
  DICTIONARY
  DISTRIBUTE
  DISTRIBUTED
  DISTRIBUTION
  DYNAMIC
  EBCDIC
  ECLCRC
  ELSE
  ELSEIF
  EMBED
  EMBEDDED
  _EMPTY_
  ENCODING
  ENCRYPT
  ENCRYPTED
  END
  ENDCPP
  ENDEMBED
  ENTH
  ENUM
  TOK_ERROR
  ESCAPE
  EVALUATE
  EVENT
  EVENTEXTRA
  EVENTNAME
  EXCEPT
  EXCLUSIVE
  EXISTS
  EXP
  EXPIRE
  EXPORT
  EXTEND
  FAIL
  FAILCODE
  FAILMESSAGE
  FAILURE
  TOK_FALSE
  FEATURE
  FETCH
  FEW
  FILEPOSITION
  FILTERED
  FIRST
  TOK_FIXED
  FLAT
  FROM
  FORMAT_ATTR
  FORWARD
  FROMUNICODE
  FROMXML
  FULL
  FUNCTION
  GETENV
  GLOBAL
  GRAPH
  GROUP
  GROUPBY
  GROUPED
  __GROUPED__
  GUARD
  HASH
  HASH32
  HASH64
  HASHMD5
  HAVING
  HEADING
  HINT
  HOLE
  HTTPCALL
  HTTPHEADER
  IF
  IFF
  IFBLOCK
  TOK_IGNORE
  IMPLEMENTS
  IMPORT
  INDEPENDENT
  INLINE
  TOK_IN
  INNER
  INTERFACE
  INTERNAL
  INTFORMAT
  ISNULL
  ISVALID
  ITERATE
  JOIN
  JOINED
  KEEP
  KEYDIFF
  KEYED
  KEYPATCH
  KEYUNICODE
  LABELED
  LAST
  LEFT
  LENGTH
  LIBRARY
  LIMIT
  LINKCOUNTED
  LITERAL
  LITTLE
  LN
  LOADXML
  LOCAL
  LOCALE
  LOCALFILEPOSITION
  TOK_LOG
  LOGICALFILENAME
  LOOKUP
  LOOP
  LZW
  MANY
  MAP
  MATCHED
  MATCHLENGTH
  MATCHPOSITION
  MATCHROW
  MATCHTEXT
  MATCHUNICODE
  MATCHUTF8
  MAX
  MAXCOUNT
  MAXLENGTH
  MAXSIZE
  MERGE
  MERGE_ATTR
  MERGEJOIN
  MIN
  MODULE
  MOFN
  MULTIPLE
  NAMED
  NAMEOF
  NAMESPACE
  NOBOUNDCHECK
  NOCASE
  NOFOLD
  NOHOIST
  NOLOCAL
  NONEMPTY
  NOOVERWRITE
  NORMALIZE
  NOROOT
  NOSCAN
  NOSORT
  __NOSTREAMING__
  NOT
  NOTHOR
  NOTIFY
  NOTRIM
  NOXPATH
  OF
  OMITTED
  ONCE
  ONFAIL
  ONLY
  ONWARNING
  OPT
  OR
  ORDERED
  OUTER
  OUTPUT
  TOK_OUT
  OVERWRITE
  __OWNED__
  PACKED
  PARALLEL
  PARSE
  PARTITION
  PARTITION_ATTR
  TOK_PATTERN
  PENALTY
  PERSIST
  PHYSICALFILENAME
  PIPE
  __PLATFORM__
  POWER
  PREFETCH
  PRELOAD
  PRIORITY
  PRIVATE
  PROCESS
  PROJECT
  PROXYADDRESS
  PULL
  PULLED
  QUOTE
  RANDOM
  RANGE
  RANK
  RANKED
  REALFORMAT
  RECORD
  RECORDOF
  RECOVERY
  REGEXFIND
  REGEXREPLACE
  REGROUP
  REJECTED
  RELATIONSHIP
  REMOTE
  REPEAT
  RESPONSE
  RETRY
  RETURN
  RIGHT
  RIGHT_NN
  ROLLUP
  ROUND
  ROUNDUP
  ROW
  ROWS
  ROWSET
  ROWDIFF
  RULE
  SAMPLE
  SCAN
  SECTION
  SELF
  SEPARATOR
  __SEQUENCE__
  SEQUENTIAL
  SERVICE
  SET
  SHARED
  SIMPLE_TYPE
  SIN
  SINGLE
  SINH
  SIZEOF
  SKEW
  SKIP
  SMART
  SOAPACTION
  SOAPCALL
  SORT
  SORTED
  SQL
  SQRT
  STABLE
  __STAND_ALONE__
  STEPPED
  STORED
  STREAMED
  SUBSORT
  SUCCESS
  SUM
  SWAPPED
  TABLE
  TAN
  TANH
  TERMINATOR
  THEN
  THISNODE
  THOR
  THRESHOLD
  TIMEOUT
  TIMELIMIT
  TOKEN
  TOPN
  TOUNICODE
  TOXML
  TRANSFER
  TRANSFORM
  TRIM
  TRUNCATE
  TOK_TRUE
  TYPE
  TYPEOF
  UNICODEORDER
  UNGROUP
  UNORDERED
  UNSIGNED
  UNSORTED
  UNSTABLE
  UPDATE
  USE
  VALIDATE
  VARIANCE
  VIRTUAL
  WAIT
  TOK_WARNING
  WHEN
  WHICH
  WIDTH
  WILD
  WITHIN
  WHOLE
  WORKUNIT
  XML_TOKEN
  XMLDECODE
  XMLDEFAULT
  XMLENCODE
  XMLPROJECT
  XMLTEXT
  XMLUNICODE
  XPATH

  
//Operators
  FIELD_REF
  FIELDS_REF
  ANDAND
  EQ
  NE
  LE
  LT
  GE
  GT
  ORDER
  ASSIGN
  GOESTO
  DOTDOT
  DIV
  SHIFTL
  SHIFTR

  DATAROW_ID
  DATASET_ID
  DICTIONARY_ID
  SCOPE_ID
  VALUE_ID
  VALUE_ID_REF
  ACTION_ID
  UNKNOWN_ID
  RECORD_ID
  ALIEN_ID
  TRANSFORM_ID
  PATTERN_ID
  FEATURE_ID
  EVENT_ID
  ENUM_ID
  LIST_DATASET_ID
  SORTLIST_ID

  TYPE_ID
  SET_TYPE_ID
  PATTERN_TYPE_ID
  DATASET_TYPE_ID
  DICTIONARY_TYPE_ID

  DATAROW_FUNCTION
  DATASET_FUNCTION
  DICTIONARY_FUNCTION
  VALUE_FUNCTION
  ACTION_FUNCTION
  PATTERN_FUNCTION
  RECORD_FUNCTION
  EVENT_FUNCTION
  SCOPE_FUNCTION
  TRANSFORM_FUNCTION
  LIST_DATASET_FUNCTION

  VALUE_MACRO
  DEFINITIONS_MACRO

  BOOL_CONST
  INTEGER_CONST
  STRING_CONST
  DATA_CONST
  REAL_CONST
  UNICODE_CONST
  TYPE_LPAREN
  TYPE_RPAREN

  MACRO
  COMPLEX_MACRO
  ENDMACRO
  SKIPPED
  HASHEND
  HASHELIF
  HASHBREAK

  INDEX
  HASH_CONSTANT
  HASH_OPTION
  HASH_WORKUNIT
  HASH_STORED
  HASH_LINK
  HASH_ONWARNING

  INTERNAL_READ_NEXT_TOKEN

//  __INTERNAL__HASHDEFINED_FOUND
//  __INTERNAL__HASHDEFINED_NOTFOUND

  /* add new token before this! */
  YY_LAST_TOKEN

%left LOWEST_PRECEDENCE
%left VALUE_MACRO
%left OR
%left AND

%left reduceAttrib

%left ORDER UNICODEORDER
%left SHIFTL SHIFTR
%left '+' '-'
%left '*' '/' '%' DIV
%left '|' '^'
%left '&' ANDAND

%left NOT

%left '.'
%left '('
%left '['
%left HIGHEST_PRECEDENCE

%%

//================================== begin of syntax section ==========================

hqlQuery
    : ENCRYPTED hqlQueryBody    { ignoreBisonWarnings3($$,$1,$2); }
    | hqlQueryBody
    ;

hqlQueryBody
    : definitions
    | query             
                        {   parser->addResult($1.getExpr(), $1); $$.clear(); }
    | definitions query 
                        {   
                            ignoreBisonWarning($1);
                            parser->addResult($2.getExpr(), $2); $$.clear();
                        }
    | RETURN goodObject ';'
                        {   parser->addResult($2.getExpr(), $2); $$.clear(); }
    | definitions setActiveToExpected RETURN goodObject ';'
                        {   parser->addResult($4.getExpr(), $4); $$.clear(); }
    | compoundModule ';'
                        {   parser->addResult($1.getExpr(), $1); $$.clear(); }
 //Temporary productions...
    | recordDef ';'                             
                        {   
                            ignoreBisonWarning($2);
                            parser->addResult($1.getExpr(), $1); $$.clear();
                        }
    | definitions recordDef ';'                 
                        {   
                            ignoreBisonWarnings2($1, $3);
                            parser->addResult($2.getExpr(), $2); $$.clear();
                        }
    |
 //Special production used processing template queries
    | GOESTO goodObject ';'                     
                        {   parser->addResult($2.getExpr(), $2); $$.clear(); }
    ;

setActiveToExpected
    :                   {   parser->setCurrentToExpected(); $$.clear(); }
    ;
    

importSection
    : startIMPORT importItem endIMPORT   
                        {   parser->lastpos = $3.pos.position+1; $$.clear(); }
    | startIMPORT error endIMPORT       
                        {   parser->lastpos = $3.pos.position+1; $$.clear(); }
    ;

startIMPORT
    : IMPORT            {   parser->setIdUnknown(true); $$.clear(); }
    ;

endIMPORT
    : ';'               {   parser->setIdUnknown(false); $$.clear(); }
    ;

importItem
    : importSelectorList
                        {
                            parser->processImport($1, NULL);
                            $$.clear();
                        }
    | importSelectorList FROM importSelector
                        {
                            parser->processImport($1, $3, NULL);
                            $$.clear();
                        }
    | importSelectorList AS UNKNOWN_ID
                        {
                            parser->processImport($1, $3.getId());
                            $$.clear();
                        }
    | importSelectorList FROM importSelector AS UNKNOWN_ID
                        {
                            parser->processImport($1, $3, $5.getId());
                            $$.clear();
                        }
    | '*' FROM importSelector
                        {
                            parser->processImportAll($3);
                            $$.clear();
                        }
    | importSelectorList AS '*'
                        {
                            if (queryLegacyImportSemantics())
                                parser->reportWarning(ERR_DEPRECATED, $1.pos, "IMPORT <module> AS * is deprecated, use IMPORT * FROM <module>");
                            else
                                parser->reportError(ERR_DEPRECATED, $1.pos, "IMPORT <module> AS * is deprecated, use IMPORT * FROM <module>");
                            parser->processImportAll($1);
                            $$.clear();
                        }
    ;

importSelectorList
    : beginList importItems
                        {
                            HqlExprArray importItems;
                            parser->endList(importItems);
                            $$.setExpr(createComma(importItems), $2);
                        }
    ;
    
importItems
    : importSelector
                        {
                            parser->addListElement($1.getExpr());
                            $$.clear();
                        }
    | importItems ',' importSelector
                        {
                            parser->addListElement($3.getExpr());
                            $$.clear();
                        }
    ;
    
importSelector
    : importId;
    

importId
    : UNKNOWN_ID        {
                            $$.setExpr(createId($1.getId()), $1);
                        }
    | '$'               {
                            $$.setExpr(createAttribute(selfAtom), $1);
                        }
    | importId '.' UNKNOWN_ID
                        {
                            $$.setExpr(createAttribute(_dot_Atom, $1.getExpr(), createId($3.getId())), $1);
                        }
    | importId '.' '^'
                        {
                            $$.setExpr(createAttribute(_container_Atom, $1.getExpr()), $1);
                        }
    ;

defineType
    : typeDef
    | setType
    | explicitDatasetType
    | explicitDictionaryType
    | ROW               {
                            IHqlExpression* record = queryNullRecord();
                            $$.setType(makeRowType(record->getType()));
                            $$.setPosition($1);
                        }
    | transformType
    ;

explicitDatasetType
    : explicitDatasetType1
    | GROUPED explicitDatasetType1
                        {
                            $$.setType(makeGroupedTableType($2.getType()));
                            $$.setPosition($1);
                        }
    ;
    
explicitDatasetType1
    : DATASET
                        {
                            $$.setType(makeTableType(makeRowType(queryNullRecord()->getType())));
                            $$.setPosition($1);
                        }
    | DATASET '(' recordDef childDatasetOptions ')'
                        {
                            OwnedHqlExpr record = $3.getExpr();
                            OwnedHqlExpr options = $4.getExpr();
                            ITypeInfo * recordType = createRecordType(record);
                            Owned<ITypeInfo> tableType = makeTableType(makeRowType(recordType));
                            if (options)
                                tableType.setown(makeAttributeModifier(LINK(tableType), createAttribute(_childAttr_Atom, LINK(options))));
                            $$.setType(tableType.getClear());
                            $$.setPosition($1);
                        }
    | _ARRAY_ explicitDatasetType
                        {
                            $$.setType(makeOutOfLineModifier($2.getType()));
                            $$.setPosition($1);
                        }
    | LINKCOUNTED explicitDatasetType
                        {
                            Owned<ITypeInfo> dsType = $2.getType();
                            $$.setType(setLinkCountedAttr(dsType, true));
                            $$.setPosition($1);
                        }
    | STREAMED explicitDatasetType
                        {
                            Owned<ITypeInfo> dsType = $2.getType();
                            Owned<ITypeInfo> linkedType = setLinkCountedAttr(dsType, true);
                            $$.setType(setStreamedAttr(linkedType, true));
                            $$.setPosition($1);
                        }
    | EMBEDDED explicitDatasetType
                        {
                            $$.setType(makeAttributeModifier($2.getType(), getEmbeddedAttr()));
                            $$.setPosition($1);
                        }
    | userTypedefDataset
    ;

explicitDictionaryType
    : DICTIONARY
                        {
                            $$.setType(makeDictionaryType(makeRowType(queryNullRecord()->getType())));
                            $$.setPosition($1);
                        }
    | DICTIONARY '(' recordDef ')'
                        {
                            OwnedHqlExpr record = $3.getExpr();
                            ITypeInfo * recordType = createRecordType(record);
                            $$.setType(makeDictionaryType(makeRowType(recordType)));
                            $$.setPosition($1);
                        }
    | LINKCOUNTED explicitDictionaryType
                        {
                            Owned<ITypeInfo> dsType = $2.getType();
                            $$.setType(setLinkCountedAttr(dsType, true));
                            $$.setPosition($1);
                        }
    | userTypedefDictionary
    ;

explicitRowType
    : explicitRowType1
    | LINKCOUNTED explicitRowType1
                        {
                            Owned<ITypeInfo> rowType = $2.getType();
                            $$.setType(setLinkCountedAttr(rowType, true));
                            $$.setPosition($1);
                        }
    ;

explicitRowType1
    : ROW               {
                            IHqlExpression* record = queryNullRecord();
                            $$.setType(makeRowType(record->getType()));
                            $$.setPosition($1);
                        }
    | ROW '(' recordDef ')'
                        {
                            OwnedHqlExpr record = $3.getExpr();
                            $$.setType(makeRowType(record->getType()));
                            $$.setPosition($1);
                        }
    ;


transformType
    : TRANSFORM '(' recordDef ')'
                        {
                            OwnedHqlExpr record = $3.getExpr();
                            $$.setType(makeTransformType(LINK(record->queryRecordType())), $1);
                        }
    | TRANSFORM '(' dataSet ')'
                        {
                            OwnedHqlExpr ds = $3.getExpr();
                            $$.setType(makeTransformType(LINK(ds->queryRecordType())), $1);
                        }
    ;
    
propType
    : simpleType           /* only simple type is supported for service */
    | scopeFlag simpleType
                        {
                            parser->reportError(ERR_SVC_NOSCOPEMODIFIER,$1,"Function in service can not specify EXPORT or SHARED");
                            $$.setType($2.getType());
                        }
    | scopeFlag     {
                            parser->reportError(ERR_SVC_NOSCOPEMODIFIER,$1,"Function in service can not specify EXPORT or SHARED");
                            $$.setType(makeVoidType());
                        }
    ;

paramType
    : typeDef
                        {
                            OwnedITypeInfo type = $1.getType();
                            //Syntactic oddity.  A record means a row of that record type.
                            if (type->getTypeCode() == type_record)
                                type.setown(makeRowType(type.getClear()));
                            $$.setType(type.getClear());
                            $$.setPosition($1);
                        }
    | DATASET_ID        {
                            OwnedHqlExpr dataset = $1.getExpr();
//                          $$.setType(makeOriginalModifier(createRecordType(dataset), LINK(dataset)));
                            $$.setType(makeRowType(createRecordType(dataset)));
                            $$.setPosition($1);
                        }
    | moduleScopeDot DATASET_ID leaveScope
                        {
                            //slightly nasty to add two original modifiers to the record, but the first one allows
                            //us to get the record right, and the second the types of parameters on transforms.
                            $1.release();
                            OwnedHqlExpr dataset = $2.getExpr();
                            //$$.setType(makeOriginalModifier(createRecordType(dataset), LINK(dataset)));
                            $$.setType(makeRowType(createRecordType(dataset)));
                            $$.setPosition($2);
                        }
    | abstractDataset   {
                            OwnedHqlExpr record = $1.getExpr();
                            OwnedHqlExpr abstractRecord = createAbstractRecord(record);
                            OwnedITypeInfo type = makeTableType(makeRowType(abstractRecord->getType()));
                            $$.setType(type.getClear(), $1);
                            parser->setTemplateAttribute();
                        }
    | explicitDatasetType
    | explicitDictionaryType
    | explicitRowType
    | abstractModule
                        {
                            OwnedHqlExpr scope = $1.getExpr();
                            $$.setType(scope->getType());       // more what about if moduleScope is a parameter?
                            $$.setPosition($1);
                        }
    | patternParamType
    | VIRTUAL RECORD
                        {
                            IHqlExpression* record = queryNullRecord();
                            OwnedHqlExpr abstractRecord = createAbstractRecord(record);
                            $$.setType(abstractRecord->getType(), $1);
                            parser->setTemplateAttribute();
                        }
    ;

patternParamType
    : TOK_PATTERN       {
                            $$.setType(makePatternType());
                            $$.setPosition($1);
                        }
    | RULE              {
                            $$.setType(makeRuleType(NULL));
                            $$.setPosition($1);
                        }
    | TOKEN             {
                            $$.setType(makeTokenType());
                            $$.setPosition($1);
                        }
    ;

object
    : goodObject
    | badObject
    ;

goodObject
    : dataSet
    | dictionary
    | expression
                        {
                            //Remove later to allow sortlist attributes
                            parser->normalizeExpression($1);
                            $$.inherit($1);
                        }
    | dataRow
    | recordDef
    | service
    | action
    | transformDef
    | transform
    | complexType
    | macro
    | embedBody
    | eventObject
    | compoundAttribute
    | abstractModule
    | goodTypeObject optFieldAttrs
                        {
                            Owned<ITypeInfo> type = $1.getType();
                            HqlExprArray attrs;
                            $2.unwindCommaList(attrs);
                            $$.setExpr(createValue(no_typedef, type.getClear(), attrs), $1);
                        }
    | enumDef
    | enumTypeId
    | setOfDatasets
    | anyFunction
    | fieldSelectedFromRecord
                        {
                            OwnedHqlExpr value = $1.getExpr();
                            $$.setExpr(createValue(no_indirect, value->getType(), LINK(value)), $1);
                        }
    | __SEQUENCE__
                        {
                            //NB: Undocumented experimental feature
                            //This is only allowed as an object, rather than a sequence so we can correctly check whether the containing
                            //attribute (if any) is parametered, and add the implicit parameter there
                            $$.setExpr(parser->createScopedSequenceExpr(), $1);
                        }
    | DEFINE goodObject
                        {
                            //Ugly internal unsupported syntax for defining an out of line function
                            //Needs a lot more work.
                            OwnedHqlExpr value = $2.getExpr();
                            $$.setExpr(parser->convertToOutOfLineFunction($2.pos, value), $1);
                        }
    ;

goodTypeObject
    : setType
    | simpleType
    | alienTypeInstance
    | userTypedefType
    | RULE TYPE '(' recordDef ')' 
                        {
                            OwnedHqlExpr record = $4.getExpr();
                            $$.setType(makeRuleType(record->getType()));
                            $$.setPosition($1);
                        }
    | explicitDatasetType
    | explicitDictionaryType
    ;

badObject
    : error             {
                            parser->processError(false);
                            $$.setExpr(createConstant((__int64) 0), $1);
                        }
    ;

macro
    : MACRO             {
                            Owned<IFileContents> contents = $1.getContents();
                            IHqlExpression* expr = createUnknown(no_macro, makeBoolType(), macroAtom, LINK(contents));
#if defined(TRACE_MACRO)
                            PrintLog("MACRO>> verify: macro definition at %d:%d\n",yylval.startLine, yylval.startColumn);
#endif

                            //Use a named symbol to associate a line number/column
                            expr = createSymbol(macroId, NULL, expr, NULL,
                                                false, false, (object_type)0,
                                                NULL,
                                                yylval.pos.lineno, yylval.pos.column, 0, 0, 0);
                            $$.setExpr(expr, $1);
                        }
    | COMPLEX_MACRO     {
                            Owned<IFileContents> contents = $1.getContents();

                            IHqlExpression* expr = createUnknown(no_macro, makeVoidType(), macroAtom, LINK(contents));

#if defined(TRACE_MACRO)
                            PrintLog("MACRO>> verify: macro definition at %d:%d\n",yylval.startLine, yylval.startColumn);
#endif

                            //Use a named symbol to associate a line number/column
                            expr = createSymbol(macroId, NULL, expr, NULL,
                                                false, false, (object_type)0,
                                                NULL,
                                                yylval.pos.lineno, yylval.pos.column, 0, 0, 0);

                            $$.setExpr(expr, $1);
                        }
    ;

embedBody
    : CPPBODY           {
                            OwnedHqlExpr embeddedCppText = $1.getExpr();
                            $$.setExpr(parser->processEmbedBody($1, embeddedCppText, NULL, NULL), $1);
                        }
    | embedPrefix CPPBODY
                        {
                            OwnedHqlExpr language = $1.getExpr();
                            OwnedHqlExpr embedText = $2.getExpr();
                            if (language->getOperator()==no_comma)
                                $$.setExpr(parser->processEmbedBody($2, embedText, language->queryChild(0), language->queryChild(1)), $1);
                            else
                                $$.setExpr(parser->processEmbedBody($2, embedText, language, NULL), $1);
                        }
    | EMBED '(' abstractModule ',' expression ')'
                        {
                            parser->normalizeExpression($5, type_stringorunicode, true);
                            OwnedHqlExpr language = $3.getExpr();
                            OwnedHqlExpr embedText = $5.getExpr();
                            $$.setExpr(parser->processEmbedBody($5, embedText, language, NULL), $1);
                        }
    | IMPORT '(' abstractModule ',' expression attribs ')'
                        {
                            parser->normalizeExpression($5, type_stringorunicode, true);
                            OwnedHqlExpr language = $3.getExpr();
                            OwnedHqlExpr funcname = $5.getExpr();
                            OwnedHqlExpr attribs = createComma(createAttribute(importAtom), $6.getExpr());
                            $$.setExpr(parser->processEmbedBody($6, funcname, language, attribs), $1);
                        }
    
    ;

embedPrefix
    : EMBED '(' abstractModule attribs ')'
                        {
                            parser->getLexer()->enterEmbeddedMode();
                            $$.setExpr(createComma($3.getExpr(), $4.getExpr()), $1);
                        }
    ;

compoundAttribute
    : startCompoundAttribute optDefinitions returnAction ';' END
                        {
                            $$.setExpr(parser->processCompoundFunction($3, false), $3);
                        }
    ;

startCompoundAttribute
    : FUNCTION
                        {
                            parser->enterScope(false);
                            parser->enterCompoundObject();
                            $$.clear();
                        }
    ;

returnAction
    : RETURN goodObject
                        {
                            $$.setExpr($2.getExpr(), $2);
                        }
    ;

compoundModule
    : startCompoundModule moduleBase moduleDefinitions END
                        {
                            $$.setExpr(parser->processModuleDefinition($1), $1);
                        }
    | PROJECT '(' abstractModule ',' abstractModule scopeProjectOpts ')'
                        {
                            OwnedHqlExpr flags = $6.getExpr();
                            IHqlExpression *expr = parser->implementInterfaceFromModule($3, $5, flags);
                            $$.setExpr(expr, $1);
                        }
    ;

startCompoundModule
    : MODULE
                        {
                            parser->enterVirtualScope();
                            parser->enterCompoundObject();
                            $$.setPosition($1);
                        }
    | INTERFACE
                        {
                            parser->enterVirtualScope();
                            parser->enterCompoundObject();
                            OwnedHqlExpr attr = createAttribute(interfaceAtom);
                            parser->appendToActiveScope(attr);
                            $$.setPosition($1);
                        }
    ;

moduleDefinitions
    :
    | moduleDefinitions moduleDefinition
    ;

moduleBase
    : '(' abstractModuleList ')' moduleOptions
    | moduleOptions
    ;

moduleOptions
    :                                                                       %prec LOWEST_PRECEDENCE     // Ensure that '(' gets shifted instead of causing a s/r error
    | moduleOptions ',' moduleOption
                        {
                            OwnedHqlExpr expr = $3.getExpr();
                            if (expr)
                                parser->appendToActiveScope(expr);
                            $$.clear();
                            $$.setPosition($3);
                        }
    ;

moduleOption
    : __NOSTREAMING__
                        {
                            //Only for internal testing...
                            $$.setExpr(createAttribute(_noStreaming_Atom), $1);
                        }
    | INTERFACE
                        {
                            $$.setExpr(createAttribute(interfaceAtom), $1);
                        }
    | VIRTUAL
                        {
                            $$.setExpr(createAttribute(virtualAtom), $1);
                        }
    | FORWARD
                        {
                            parser->processForwardModuleDefinition($1);
                            $$.setExpr(NULL, $1);
                        }
    | LIBRARY '(' scopeFunction ')'
                        {
                            $$.setExpr(createExprAttribute(libraryAtom, $3.getExpr()), $1);
                        }
    ;

abstractModuleList
    : abstractModuleItem
    | abstractModuleList ',' abstractModuleItem
    ;

abstractModuleItem
    : abstractModule
                        {
                            OwnedHqlExpr expr = $1.getExpr();
                            if (parser->checkValidBaseModule($1, expr))
                                parser->appendToActiveScope(expr);

                            $$.clear();
                            $$.setPosition($1);
                        }
    ;

//MORE: If complex type could be defined in a non assignment context then TYPE will need to become hard reserved.
complexType
    : startTYPE definitions END
                        {
                            $$.setExpr(parser->processAlienType($3),$1);
                        }
    | startTYPE END     {
                            parser->reportError(ERR_USRTYPE_EMPTYDEF,$1,"Empty user TYPE definition");
                            $$.setExpr(parser->processAlienType($2),$1);
                        }
    ;

startTYPE
    : TYPE              {
                            parser->beginAlienType($1);
                            $$.clear();
                            $$.setPosition($1);
                        }
    ;

defineid
    : UNKNOWN_ID        {
                            parser->beginDefineId($1.getId(), NULL);
                            $$.setType(NULL);
                            $$.setPosition($1);
                        }
    | SCOPE_ID          {
                            parser->beginDefineId(parser->getNameFromExpr($1), NULL);
                            $$.setType(NULL);
                            $$.setPosition($1);
                        }
    | recordDef         {
                            parser->beginDefineId(parser->getNameFromExpr($1), NULL);
                            $$.setType(NULL);
                            $$.setPosition($1);
                        }
    | TRANSFORM_ID      {
                            parser->beginDefineId(parser->getNameFromExpr($1), NULL);
                            $$.setType(NULL);
                            $$.setPosition($1);
                        }
    | TYPE_ID           {
                            parser->beginDefineId(parser->getNameFromExpr($1), NULL);
                            $$.setType(NULL);
                            $$.setPosition($1);
                        }
    | defineType knownOrUnknownId   
                        {
                            Owned<ITypeInfo> type = $1.getType();
                            if (type->getTypeCode() == type_alien)
                                type.set(type->queryPromotedType());
                            parser->beginDefineId($2.getId(), type);
                            $$.setType(type.getClear());
                            $$.setPosition($1);
                        }
    | globalScopedDatasetId knownOrUnknownId
                        {
                            OwnedHqlExpr ds = $1.getExpr();
                            parser->beginDefineId($2.getId(), ds->queryType());
                            $$.setType(ds->getType());
                            $$.setPosition($1);
                        }
    | UNKNOWN_ID UNKNOWN_ID
                        {
                            parser->reportError(ERR_UNKNOWN_TYPE, $1, "Unknown type '%s'", $1.getId()->str());
                            parser->beginDefineId($2.getId(), NULL);
                            $$.setType(NULL);
                            $$.setPosition($1);
                        }
    ;


knownOrUnknownId
    : UNKNOWN_ID
    | knownId           { $$.setId(parser->getNameFromExpr($1)); $$.setPosition($1); }
    | knownFunction1    { $$.setId(parser->getNameFromExpr($1)); $$.setPosition($1); }
    ;

knownId
    : DATAROW_ID
    | DATASET_ID
    | DICTIONARY_ID
    | VALUE_ID
    | ACTION_ID
    | RECORD_ID
    | ALIEN_ID
    | TYPE_ID
    | TRANSFORM_ID
    | FEATURE_ID
    | SCOPE_ID
    | PATTERN_ID
    | LIST_DATASET_ID
    ;

knownFunction1
    : DATAROW_FUNCTION
    | DATASET_FUNCTION
    | DICTIONARY_FUNCTION
    | VALUE_FUNCTION
    | ACTION_FUNCTION
    | PATTERN_FUNCTION
    | EVENT_FUNCTION
    | TRANSFORM_FUNCTION
    | LIST_DATASET_FUNCTION
//  Following cause s/r problems

//  | RECORD_FUNCTION
//  | SCOPE_FUNCTION
    ;


scopeFlag
    : EXPORT            {   $$.setInt(EXPORT_FLAG); $$.setPosition($1); }
    | SHARED            {   $$.setInt(SHARED_FLAG); $$.setPosition($1); }
    | LOCAL             {   $$.setInt(0); $$.setPosition($1); }
    | EXPORT VIRTUAL    {   $$.setInt(EXPORT_FLAG|VIRTUAL_FLAG); $$.setPosition($1); }
    | SHARED VIRTUAL    {   $$.setInt(SHARED_FLAG|VIRTUAL_FLAG); $$.setPosition($1); }
    ;

// scopeflags needs to be explicitly included, rather than using an optScopeFlags production, otherwise you get shift reduce errors - since it is the first item on a line.

defineidWithOptScope
    : defineid          {
                            $$.setDefineId(parser->createDefineId(0, $1.getType()));
                            $$.setPosition($1);
                        }
    | scopeFlag defineid
                        {
                            $$.setDefineId(parser->createDefineId((int)$1.getInt(), $2.getType()));
                            $$.setPosition($1);
                        }
    ;

definePatternIdWithOptScope
    : definePatternId   {
                            $$.setDefineId(parser->createDefineId(0, $1.getType()));
                            $$.setPosition($1);
                        }
    | scopeFlag definePatternId  
                        {
                            $$.setDefineId(parser->createDefineId((int)$1.getInt(), $2.getType()));
                            $$.setPosition($1);
                        }
    ;

definePatternId
    : TOK_PATTERN knownOrUnknownId  
                        {
                            ITypeInfo *type = makePatternType();
                            parser->beginDefineId($2.getId(), type);
                            $$.setType(type);
                            $$.setPosition($1);
                        }
    | RULE knownOrUnknownId 
                        {
                            ITypeInfo *type = makeRuleType(NULL);
                            parser->beginDefineId($2.getId(), type);
                            $$.setType(type);
                            $$.setPosition($1);
                        }
    | RULE '(' recordDef ')' knownOrUnknownId
                        {
                            OwnedHqlExpr record = $3.getExpr();
                            ITypeInfo *type = makeRuleType(record->getType());
                            parser->beginDefineId($5.getId(), type);
                            $$.setType(type);
                            $$.setPosition($1);
                        }
    | TOKEN knownOrUnknownId    
                        {
                            ITypeInfo *type = makeTokenType();
                            parser->beginDefineId($2.getId(), type);
                            $$.setType(type);
                            $$.setPosition($1);
                        }
    | userTypedefPattern knownOrUnknownId
                        {
                            ITypeInfo *type = $1.getType();
                            parser->beginDefineId($2.getId(), type);
                            $$.setType(type);
                            $$.setPosition($1);
                        }
    ;


optDefinitions
    :
    | optDefinitions definition 
    ;

definitions
    : definition
    | definitions definition
    ;

attributeDefinition
    : DEFINITIONS_MACRO definition
    | moduleScopeDot DEFINITIONS_MACRO leaveScope definition
                        {
                            $1.release();
                            $$.clear();
                        }

    | defineidWithOptScope parmdef ASSIGN object optfailure ';'
                        {
                            if ($5.queryExpr())
                                parser->normalizeExpression($4);
                            $$.clear($1);
                            parser->defineSymbolProduction($1, $2, $3, &$4, &$5, $6);
                        }
    | definePatternIdWithOptScope parmdef featureParameters ASSIGN pattern optfailure ';'
                        {
                            parser->definePatternSymbolProduction($1, $4, $5, $6, $7);
                            $$.clear();
                        }
    | defineFeatureIdWithOptScope ';'
                        {
                            DefineIdSt* defineid = $1.getDefineId();
                            IHqlExpression *expr = createValue(no_null, makeFeatureType());
                            expr = createValue(no_pat_featuredef, expr->getType(), expr);

                            parser->doDefineSymbol(defineid, expr, NULL, $1, $2.pos.position, $2.pos.position, false);
                            $$.clear();
                        }
    | defineFeatureIdWithOptScope ASSIGN featureDefine ';'
                        {
                            DefineIdSt* defineid = $1.getDefineId();
                            IHqlExpression *expr = $3.getExpr();
                            expr = createValue(no_pat_featuredef, expr->getType(), expr);

                            parser->doDefineSymbol(defineid, expr, NULL, $1, $2.pos.position, $4.pos.position, false);
                            $$.clear();
                        }
    ;


definition
    : simpleDefinition
    | ';'               {
                            //Extra ';' are ignored, partly to reduce problems with trailing ';' inside macros
                            $$.clear();
                        }
    ;

simpleDefinition
    : attributeDefinition
    | query ';'         {
                            parser->addResult($1.getExpr(), $1);
                            $$.clear();
                        }
 /* general error */
    | error   ';'       {
                            yyerrok;
                            parser->processError(true);
                            $$.clear();
                        }
    | importSection
    | metaCommandWithNoSemicolon simpleDefinition
    ;

metaCommandWithNoSemicolon
    : setMetaCommand
                        {
                            //These are really treated like actions now, this is here for backward compatibility
                            parser->reportWarning(ERR_DEPRECATED, $1.pos, "#command with no trailing semicolon is deprecated");
                            parser->addResult($1.getExpr(), $1);
                            $$.clear();
                        }
   ;

moduleDefinition
    : defineidWithOptScope parmdef ';'
                        {
                            parser->defineSymbolProduction($1, $2, $3, NULL, NULL, $3);
                            $$.clear();
                            $$.setPosition($1);
                        }
    | definition
    ;


setMetaCommand
    : HASH_OPTION '(' expression ',' expression ')'
                        {
                            parser->normalizeExpression($3, type_string, true);
                            parser->normalizeExpression($5, type_any, true);
                            $$.setExpr(createValue(no_setmeta, makeVoidType(), createAttribute(debugAtom), $3.getExpr(), $5.getExpr()), $1);
                        }
    | HASH_WORKUNIT '(' expression ',' expression ')'
                        {
                            parser->normalizeExpression($3, type_string, true);
                            parser->normalizeExpression($5, type_any, true);
                            $$.setExpr(createValue(no_setmeta, makeVoidType(), createAttribute(workunitAtom), $3.getExpr(), $5.getExpr()), $1);
                        }
    | HASH_STORED '(' expression ',' hashStoredValue ')'
                        {
                            parser->normalizeStoredNameExpression($3);
                            $$.setExpr(createValue(no_setmeta, makeVoidType(), createAttribute(storedAtom), $3.getExpr(), $5.getExpr()), $1);
                        }
    | HASH_CONSTANT '(' expression ',' hashStoredValue ')'
                        {
                            parser->normalizeStoredNameExpression($3);
                            $$.setExpr(createValue(no_setmeta, makeVoidType(), createAttribute(constAtom), $3.getExpr(), $5.getExpr()), $1);
                        }
    | HASH_LINK '(' expression ')'
                        {
                            parser->normalizeExpression($3, type_string, true);
                            $$.setExpr(createValue(no_setmeta, makeVoidType(), createAttribute(linkAtom), $3.getExpr()), $1);
                        }
    | HASH_ONWARNING '(' expression ',' warningAction ')'
                        {
                            parser->normalizeExpression($3, type_int, false);
                            $$.setExpr(createValue(no_setmeta, makeVoidType(), createAttribute(onWarningAtom), $3.getExpr(), $5.getExpr()), $1);
                        }
    ;

hashStoredValue
    : expression
                        {
                            parser->normalizeExpression($1, type_any, false);
                            $$.inherit($1);
                        }
    | dataSet
    | dataRow
    ;

optfailure
    : ':' failclause        
                        {
                            $$.setExpr($2.getExpr(), $1); 
                        }
    |                   {   $$.setNullExpr(); $$.clearPosition(); }
    ;

failclause
    : failure
    | failclause ',' failure
                        {
                            IHqlExpression * previousWorkflow = $1.getExpr();
                            IHqlExpression * newWorkflow = $3.getExpr();
                            parser->checkWorkflowMultiples(previousWorkflow, newWorkflow, $3);
                            $$.setExpr(createComma(previousWorkflow, newWorkflow), $1);
                        }
    ;

failure
    : FAILURE '(' action ')'
                        {
                            $$.setExpr(createValue(no_failure, $3.getExpr(), NULL), $1);
                        }
    | SUCCESS '(' action ')'
                        {
                            $$.setExpr(createValue(no_success, $3.getExpr(), NULL), $1);
                        }
    | RECOVERY '(' action ')'
                        {
                            $$.setExpr(createValue(no_recovery, $3.getExpr(), createConstant(1)), $1);
                        }
    | RECOVERY '(' action ',' expression ')'
                        {
                            parser->normalizeExpression($5, type_int, true);
                            $$.setExpr(createValue(no_recovery, $3.getExpr(), $5.getExpr()), $1);
                        }
    | WHEN '(' event ')'
                        {
                            parser->checkConstantEvent($3);
                            $$.setExpr(createValue(no_when, $3.getExpr()), $1);
                        }
    | WHEN '(' event ',' COUNT '(' expression ')'  ')'
                        {
                            parser->checkConstantEvent($3);
                            parser->normalizeExpression($7, type_int, true);
                            $$.setExpr(createValue(no_when, $3.getExpr(), $7.getExpr()), $1);
                        }
    | PRIORITY '(' expression ')'
                        {
                            parser->normalizeExpression($3, type_int, true);
                            $$.setExpr(createValue(no_priority, $3.getExpr()), $1);
                        }
    | PERSIST '(' expression ')'
                        {
                            parser->normalizeExpression($3, type_string, true);
                            $$.setExpr(createValueF(no_persist, makeVoidType(), $3.getExpr(), NULL), $1);
                        }
    | PERSIST '(' expression ',' persistOpts ')'
                        {
                            parser->normalizeExpression($3, type_string, true);
                            $$.setExpr(createValueF(no_persist, makeVoidType(), $3.getExpr(), $5.getExpr(), NULL), $1);
                        }
    | PERSIST '(' expression ',' expression optPersistOpts ')'
                        {
                            parser->normalizeExpression($3, type_string, true);
                            parser->normalizeExpression($5, type_string, true);
                            $$.setExpr(createValueF(no_persist, makeVoidType(), $3.getExpr(), $5.getExpr(), $6.getExpr(), NULL), $1);
                        }
    | STORED '(' expression optFewMany ')'
                        {
                            parser->normalizeStoredNameExpression($3);
                            $$.setExpr(createValue(no_stored, makeVoidType(), $3.getExpr(), $4.getExpr()), $1);
                        }
    | CHECKPOINT '(' constExpression ')'
                        {
                            parser->normalizeStoredNameExpression($3);
                            $$.setExpr(createValue(no_checkpoint, makeVoidType(), $3.getExpr()), $1);
                        }
    | GLOBAL                    
                        {
                            $$.setExpr(createValue(no_global), $1);
                        }
    | GLOBAL '(' fewMany ')'
                        {
                            $$.setExpr(createValue(no_global, $3.getExpr()), $…