// AsmJit - Complete JIT Assembler for C++ Language.

// Copyright (c) 2008-2010, Petr Kobalicek <kobalicek.petr@gmail.com>
//
// Permission is hereby granted, free of charge, to any person
// obtaining a copy of this software and associated documentation
// files (the "Software"), to deal in the Software without
// restriction, including without limitation the rights to use,
// copy, modify, merge, publish, distribute, sublicense, and/or sell
// copies of the Software, and to permit persons to whom the
// Software is furnished to do so, subject to the following
// conditions:
//
// The above copyright notice and this permission notice shall be
// included in all copies or substantial portions of the Software.
//
// THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND,
// EXPRESS OR IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES
// OF MERCHANTABILITY, FITNESS FOR A PARTICULAR PURPOSE AND
// NONINFRINGEMENT. IN NO EVENT SHALL THE AUTHORS OR COPYRIGHT
// HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER LIABILITY,
// WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING
// FROM, OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR
// OTHER DEALINGS IN THE SOFTWARE.

// [Guard]
#ifndef _ASMJIT_COMPILERX86X64_H
#define _ASMJIT_COMPILERX86X64_H

#if !defined(_ASMJIT_COMPILER_H)
#warning "AsmJit/CompilerX86X64.h can be only included by AsmJit/Compiler.h"
#endif // _ASMJIT_COMPILER_H

// [Dependencies]
#include "Build.h"
#include "Assembler.h"
#include "Defs.h"
#include "Operand.h"
#include "Util.h"

#include <string.h>

// A little bit C++.
#include <new>

// [Api-Begin]
#include "ApiBegin.h"

//! @internal
//!
//! @brief Mark methods not supported by @ref Compiler. These methods are
//! usually used only in function prologs/epilogs or to manage stack.
#define ASMJIT_NOT_SUPPORTED_BY_COMPILER 0

namespace AsmJit {

//! @addtogroup AsmJit_Compiler
//! @{

// ============================================================================
// [Forward Declarations]
// ============================================================================

struct CodeGenerator;

// ============================================================================
// [AsmJit::TypeToId]
// ============================================================================

// Skip documenting this.
#if !defined(ASMJIT_NODOC)

ASMJIT_DECLARE_TYPE_AS_ID(int8_t, VARIABLE_TYPE_GPD);
ASMJIT_DECLARE_TYPE_AS_ID(uint8_t, VARIABLE_TYPE_GPD);

ASMJIT_DECLARE_TYPE_AS_ID(int16_t, VARIABLE_TYPE_GPD);
ASMJIT_DECLARE_TYPE_AS_ID(uint16_t, VARIABLE_TYPE_GPD);

ASMJIT_DECLARE_TYPE_AS_ID(int32_t, VARIABLE_TYPE_GPD);
ASMJIT_DECLARE_TYPE_AS_ID(uint32_t, VARIABLE_TYPE_GPD);

#if defined(ASMJIT_X64)
ASMJIT_DECLARE_TYPE_AS_ID(int64_t, VARIABLE_TYPE_GPQ);
ASMJIT_DECLARE_TYPE_AS_ID(uint64_t, VARIABLE_TYPE_GPQ);
#endif // ASMJIT_X64

ASMJIT_DECLARE_TYPE_AS_ID(float, VARIABLE_TYPE_FLOAT);
ASMJIT_DECLARE_TYPE_AS_ID(double, VARIABLE_TYPE_DOUBLE);

#endif // !ASMJIT_NODOC

// ============================================================================
// [AsmJit::FunctionPrototype]
// ============================================================================

//! @brief Calling convention and function argument handling.
struct ASMJIT_API FunctionPrototype
{
  // --------------------------------------------------------------------------
  // [Construction / Destruction]
  // --------------------------------------------------------------------------

  //! @brief Create a new @ref FunctionPrototype instance.
  FunctionPrototype() ASMJIT_NOTHROW;
  //! @brief Destroy the @ref FunctionPrototype instance.
  ~FunctionPrototype() ASMJIT_NOTHROW;

  // --------------------------------------------------------------------------
  // [Argument]
  // --------------------------------------------------------------------------

  //! @brief Function argument location.
  struct Argument
  {
    //! @brief Variable type, see @c VARIABLE_TYPE.
    uint32_t variableType;
    //! @brief Register index if argument is passed through register, otherwise
    //! @c INVALID_VALUE.
    uint32_t registerIndex;
    //! @brief Stack offset if argument is passed through stack, otherwise
    //! @c INVALID_VALUE.
    int32_t stackOffset;

    //! @brief Get whether the argument is assigned, for private use only.
    inline bool isAssigned() const ASMJIT_NOTHROW
    { return registerIndex != INVALID_VALUE || stackOffset != (int32_t)INVALID_VALUE; }
  };

  // --------------------------------------------------------------------------
  // [Methods]
  // --------------------------------------------------------------------------

  //! @brief Set function prototype.
  //!
  //! This will set function calling convention and setup arguments variables.
  //!
  //! @note This function will allocate variables, it can be called only once.
  void setPrototype(
    uint32_t callingConvention,
    const uint32_t* arguments, 
    uint32_t argumentsCount,
    uint32_t returnValue) ASMJIT_NOTHROW;

  //! @brief Get function calling convention, see @c CALL_CONV.
  inline uint32_t getCallingConvention() const ASMJIT_NOTHROW { return _callingConvention; }

  //! @brief Get whether the callee pops the stack.
  inline uint32_t getCalleePopsStack() const ASMJIT_NOTHROW { return _calleePopsStack; }

  //! @brief Get function arguments.
  inline Argument* getArguments() ASMJIT_NOTHROW { return _arguments; }
  //! @brief Get function arguments (const version).
  inline const Argument* getArguments() const ASMJIT_NOTHROW { return _arguments; }

  //! @brief Get count of arguments.
  inline uint32_t getArgumentsCount() const ASMJIT_NOTHROW { return _argumentsCount; }

  //! @brief Get function return value or @ref INVALID_VALUE if it's void.
  inline uint32_t getReturnValue() const ASMJIT_NOTHROW { return _returnValue; }

  //! @brief Get direction of arguments passed on the stack.
  //!
  //! Direction should be always @c ARGUMENT_DIR_RIGHT_TO_LEFT.
  //!
  //! @note This is related to used calling convention, it's not affected by
  //! number of function arguments or their types.
  inline uint32_t getArgumentsDirection() const ASMJIT_NOTHROW { return _argumentsDirection; }

  //! @brief Get stack size needed for function arguments passed on the stack.
  inline uint32_t getArgumentsStackSize() const ASMJIT_NOTHROW { return _argumentsStackSize; }

  //! @brief Get registers used to pass first integer parameters by current
  //! calling convention.
  //!
  //! @note This is related to used calling convention, it's not affected by
  //! number of function arguments or their types.
  inline const uint32_t* getArgumentsGPList() const ASMJIT_NOTHROW { return _argumentsGPList; }

  //! @brief Get registers used to pass first SP-FP or DP-FPparameters by
  //! current calling convention.
  //!
  //! @note This is related to used calling convention, it's not affected by
  //! number of function arguments or their types.
  inline const uint32_t* getArgumentsXMMList() const ASMJIT_NOTHROW { return _argumentsXMMList; }

  //! @brief Get bitmask of GP registers which might be used for arguments.
  inline uint32_t getArgumentsGP() const ASMJIT_NOTHROW { return _argumentsGP; }
  //! @brief Get bitmask of MM registers which might be used for arguments.
  inline uint32_t getArgumentsMM() const ASMJIT_NOTHROW { return _argumentsMM; }
  //! @brief Get bitmask of XMM registers which might be used for arguments.
  inline uint32_t getArgumentsXMM() const ASMJIT_NOTHROW { return _argumentsXMM; }

  //! @brief Get bitmask of general purpose registers that's preserved
  //! (non-volatile).
  //!
  //! @note This is related to used calling convention, it's not affected by
  //! number of function arguments or their types.
  inline uint32_t getPreservedGP() const ASMJIT_NOTHROW { return _preservedGP; }

  //! @brief Get bitmask of MM registers that's preserved (non-volatile).
  //!
  //! @note No standardized calling function is not preserving MM registers.
  //! This member is here for extension writers who need for some reason custom
  //! calling convention that can be called through code generated by AsmJit
  //! (or other runtime code generator).
  inline uint32_t getPreservedMM() const ASMJIT_NOTHROW { return _preservedMM; }

  //! @brief Return bitmask of XMM registers that's preserved (non-volatile).
  //!
  //! @note This is related to used calling convention, it's not affected by
  //! number of function arguments or their types.
  inline uint32_t getPreservedXMM() const ASMJIT_NOTHROW { return _preservedXMM; }

  //! @brief Get mask of all GP registers used to pass function arguments.
  inline uint32_t getPassedGP() const ASMJIT_NOTHROW { return _passedGP; }
  //! @brief Get mask of all MM registers used to pass function arguments.
  inline uint32_t getPassedMM() const ASMJIT_NOTHROW { return _passedMM; }
  //! @brief Get mask of all XMM registers used to pass function arguments.
  inline uint32_t getPassedXMM() const ASMJIT_NOTHROW { return _passedXMM; }

  //! @brief Find argument (id) by the register code. Used mainly by @ref ECall
  //! emittable.
  uint32_t findArgumentByRegisterCode(uint32_t regCode) const ASMJIT_NOTHROW;

protected:

  // --------------------------------------------------------------------------
  // [Private]
  // --------------------------------------------------------------------------

  void _clear() ASMJIT_NOTHROW;
  void _setCallingConvention(uint32_t callingConvention) ASMJIT_NOTHROW;
  void _setPrototype(
    const uint32_t* arguments,
    uint32_t argumentsCount,
    uint32_t returnValue) ASMJIT_NOTHROW;
  void _setReturnValue(uint32_t valueId) ASMJIT_NOTHROW;

  // --------------------------------------------------------------------------
  // [Members]
  // --------------------------------------------------------------------------

  //! @brief Calling convention.
  uint32_t _callingConvention;
  //! @brief Whether callee pops stack.
  uint32_t _calleePopsStack;

  //! @brief List of arguments, their register codes or stack locations.
  Argument _arguments[FUNC_MAX_ARGS];

  //! @brief Function return value.
  uint32_t _returnValue;

  //! @brief Count of arguments (in @c _argumentsList).
  uint32_t _argumentsCount;
  //! @brief Direction for arguments passed on the stack, see @c ARGUMENT_DIR.
  uint32_t _argumentsDirection;
  //! @brief Count of bytes consumed by arguments on the stack.
  uint32_t _argumentsStackSize;

  //! @brief List of registers that's used for first GP arguments.
  uint32_t _argumentsGPList[16];
  //! @brief List of registers that's used for first XMM arguments.
  uint32_t _argumentsXMMList[16];

  //! @brief Bitmask for preserved GP registers.
  uint32_t _argumentsGP;
  //! @brief Bitmask for preserved MM registers.
  uint32_t _argumentsMM;
  //! @brief Bitmask for preserved XMM registers.
  uint32_t _argumentsXMM;

  //! @brief Bitmask for preserved GP registers.
  uint32_t _preservedGP;
  //! @brief Bitmask for preserved MM registers.
  uint32_t _preservedMM;
  //! @brief Bitmask for preserved XMM registers.
  uint32_t _preservedXMM;

  // Set by _setPrototype().

  //! @brief Bitmask for GP registers used as function arguments.
  uint32_t _passedGP;
  //! @brief Bitmask for GP registers used as function arguments.
  uint32_t _passedMM;
  //! @brief Bitmask for GP registers used as function arguments.
  uint32_t _passedXMM;
};

// ============================================================================
// [AsmJit::VarData]
// ============================================================================

//! @brief Variable data (used internally by @c Compiler).
struct VarData
{
  // --------------------------------------------------------------------------
  // [Scope]
  // --------------------------------------------------------------------------

  //! @brief Scope (NULL if variable is global).
  EFunction* scope;

  //! @brief The first emittable where the variable is accessed.
  //!
  //! @note If this member is @c NULL then variable is unused.
  Emittable* firstEmittable;
  //! @brief The first callable (ECall) which is after the @c firstEmittable.
  ECall* firstCallable;
  //! @brief The last emittable where the variable is accessed.
  Emittable* lastEmittable;

  // --------------------------------------------------------------------------
  // [Id / Name]
  // --------------------------------------------------------------------------

  //! @brief Variable name.
  const char* name;
  //! @brief Variable id.
  uint32_t id;
  //! @brief Variable type.
  uint32_t type;
  //! @brief Variable size.
  uint32_t size;

  // --------------------------------------------------------------------------
  // [Home]
  // --------------------------------------------------------------------------

  //! @brief Home register index or @c INVALID_VALUE (used by register allocator).
  uint32_t homeRegisterIndex;
  //! @brief Preferred register index.
  uint32_t prefRegisterMask;

  //! @brief Home memory address offset.
  int32_t homeMemoryOffset;
  //! @brief Used by @c CompilerContext, do not touch (NULL when created).
  void* homeMemoryData;

  // --------------------------------------------------------------------------
  // [Actual]
  // --------------------------------------------------------------------------

  //! @brief Actual register index (connected with actual @c StateData).
  uint32_t registerIndex;
  //! @brief Actual working offset. This member is set before register allocator
  //! is called. If workOffset is same as CompilerContext::_currentOffset then
  //! this variable is probably used in next instruction and can't be spilled.
  uint32_t workOffset;

  //! @brief Next active variable in circular double-linked list.
  VarData* nextActive;
  //! @brief Previous active variable in circular double-linked list.
  VarData* prevActive;

  // --------------------------------------------------------------------------
  // [Flags]
  // --------------------------------------------------------------------------

  //! @brief Variable priority.
  uint8_t priority;
  //! @brief Whether variable content can be calculated by simple instruction
  //!
  //! This is used mainly by mmx or sse2 code and variable allocator will
  //! never reserve space for this variable. Calculated variables are for
  //! example all zeros, all ones, etc.
  uint8_t calculated;
  //! @brief Whether variable is argument passed through register.
  uint8_t isRegArgument;
  //! @brief Whether variable is argument passed through memory.
  uint8_t isMemArgument;

  //! @brief Variable state (connected with actual @c StateData).
  uint8_t state;
  //! @brief Whether variable was changed (connected with actual @c StateData).
  uint8_t changed;
  //! @brief Save on unuse (at end of the variable scope).
  uint8_t saveOnUnuse;

  // --------------------------------------------------------------------------
  // [Statistics]
  // --------------------------------------------------------------------------

  //! @brief Register read statistics (used by instructions where this variable needs
  //! to be read only).
  uint32_t registerReadCount;
  //! @brief Register write statistics (used by instructions where this variable needs
  //! to be write only).
  uint32_t registerWriteCount;
  //! @brief Register read+write statistics (used by instructions where this variable
  //! needs to be read and write).
  uint32_t registerRWCount;

  //! @brief Register GPB.LO statistics (for code generator).
  uint32_t registerGPBLoCount;
  //! @brief Register GPB.HI statistics (for code generator).
  uint32_t registerGPBHiCount;

  //! @brief Memory read statistics.
  uint32_t memoryReadCount;
  //! @brief Memory write statistics.
  uint32_t memoryWriteCount;
  //! @brief Memory read+write statistics.
  uint32_t memoryRWCount;

  // --------------------------------------------------------------------------
  // [Temporary]
  // --------------------------------------------------------------------------

  //! @brief Temporary data that can be used in prepare/translate stage.
  //!
  //! Initial value is NULL and each emittable/code that will use it must also
  //! clear it.
  //!
  //! This temporary data is designed to be used by algorithms that need to
  //! set some state into the variables, do something and then cleanup. See
  //! state-switch and function call.
  union
  {
    void* tempPtr;
    sysint_t tempInt;
  };
};

// ============================================================================
// [AsmJit::VarMemBlock]
// ============================================================================

struct VarMemBlock
{
  int32_t offset;
  uint32_t size;

  VarMemBlock* nextUsed;
  VarMemBlock* nextFree;
};

// ============================================================================
// [AsmJit::VarAllocRecord]
// ============================================================================

//! @brief Variable alloc record (for each instruction that uses variables).
//!
//! Variable record contains pointer to variable data and register allocation
//! flags. These flags are important to determine the best alloc instruction.
struct VarAllocRecord
{
  //! @brief Variable data (the structure owned by @c Compiler).
  VarData* vdata;
  //! @brief Variable alloc flags, see @c VARIABLE_ALLOC.
  uint32_t vflags;
  //! @brief Register mask (default is 0).
  uint32_t regMask;
};

// ============================================================================
// [AsmJit::VarCallRecord]
// ============================================================================

//! @brief Variable call-fn record (for each callable that uses variables).
//!
//! This record contains variables that are used to call a function (using 
//! @c ECall emittable). Each variable contains the registers where it must
//! be and registers where the value will be returned.
struct VarCallRecord
{
  //! @brief Variable data (the structure owned by @c Compiler).
  VarData* vdata;
  uint32_t flags;

  uint8_t inCount;
  uint8_t inDone;

  uint8_t outCount;
  uint8_t outDone;

  enum FLAGS
  {
    FLAG_IN_GP = 0x0001,
    FLAG_IN_MM = 0x0002,
    FLAG_IN_XMM = 0x0004,
    FLAG_IN_STACK = 0x0008,

    FLAG_OUT_EAX = 0x0010,
    FLAG_OUT_EDX = 0x0020,
    FLAG_OUT_ST0 = 0x0040,
    FLAG_OUT_ST1 = 0x0080,
    FLAG_OUT_MM0 = 0x0100,
    FLAG_OUT_XMM0 = 0x0400,
    FLAG_OUT_XMM1 = 0x0800,

    FLAG_IN_MEM_PTR = 0x1000,
    FLAG_CALL_OPERAND_REG = 0x2000,
    FLAG_CALL_OPERAND_MEM = 0x4000,

    FLAG_UNUSE_AFTER_USE = 0x8000
  };
};

// ============================================================================
// [AsmJit::VarHintRecord]
// ============================================================================

struct VarHintRecord
{
  VarData* vdata;
  uint32_t hint;
};

// ============================================================================
// [AsmJit::StateData]
// ============================================================================

//! @brief State data.
struct StateData
{
  enum { NUM_REGS = 16 + 8 + 16 };

  inline void clear() ASMJIT_NOTHROW
  {
    memset(this, 0, sizeof(*this));
  }

  // --------------------------------------------------------------------------
  // [Members]
  // --------------------------------------------------------------------------

  union
  {
    //! @brief All allocated variables in one array.
    VarData* regs[NUM_REGS];

    struct
    {
      //! @brief Allocated GP registers.
      VarData* gp[16];
      //! @brief Allocated MM registers.
      VarData* mm[8];
      //! @brief Allocated XMM registers.
      VarData* xmm[16];
    };
  };

  //! @brief Used GP registers bitmask.
  uint32_t usedGP;
  //! @brief Used MM registers bitmask.
  uint32_t usedMM;
  //! @brief Used XMM registers bitmask.
  uint32_t usedXMM;

  //! @brief Changed GP registers bitmask.
  uint32_t changedGP;
  //! @brief Changed MM registers bitmask.
  uint32_t changedMM;
  //! @brief Changed XMM registers bitmask.
  uint32_t changedXMM;

  //! @brief Count of variables in @c memVarsData.
  uint32_t memVarsCount;
  //! @brief Variables stored in memory (@c VARIABLE_STATE_MEMORY).
  //!
  //! When saving / restoring state it's important to keep registers which are
  //! still in memory. Register is always unused when it is going out-of-scope.
  //! All variables which are not here are unused (@c VARIABLE_STATE_UNUSED).
  VarData* memVarsData[1];
};

// ============================================================================
// [AsmJit::ForwardJumpData]
// ============================================================================

struct ForwardJumpData
{
  EJmp* inst;
  StateData* state;
  ForwardJumpData* next;
};

// ============================================================================
// [AsmJit::EVariableHint]
// ============================================================================

//! @brief Variable hint.
struct ASMJIT_API EVariableHint : public Emittable
{
  // --------------------------------------------------------------------------
  // [Construction / Destruction]
  // --------------------------------------------------------------------------

  //! @brief Create a new @ref EVariableHint instance.
  EVariableHint(Compiler* c, VarData* vdata, uint32_t hintId, uint32_t hintValue) ASMJIT_NOTHROW;
  //! @brief Destroy the @ref EVariableHInt instance.
  virtual ~EVariableHint() ASMJIT_NOTHROW;

  // --------------------------------------------------------------------------
  // [Emit]
  // --------------------------------------------------------------------------

  virtual void prepare(CompilerContext& cc) ASMJIT_NOTHROW;
  virtual Emittable* translate(CompilerContext& cc) ASMJIT_NOTHROW;

  // --------------------------------------------------------------------------
  // [Utilities]
  // --------------------------------------------------------------------------

  virtual int getMaxSize() const ASMJIT_NOTHROW;

  // --------------------------------------------------------------------------
  // [Hint]
  // --------------------------------------------------------------------------

  //! @brief Get assigned variable (data).
  inline VarData* getVar() const ASMJIT_NOTHROW { return _vdata; }

  //! @brief Get hint it (see @ref VARIABLE_HINT).
  inline uint32_t getHintId() const ASMJIT_NOTHROW { return _hintId; }
  //! @brief Get hint value.
  inline uint32_t getHintValue() const ASMJIT_NOTHROW { return _hintValue; }

  //! @brief Set hint it (see @ref VARIABLE_HINT).
  inline void setHintId(uint32_t hintId) ASMJIT_NOTHROW { _hintId = hintId; }
  //! @brief Set hint value.
  inline void setHintValue(uint32_t hintValue) ASMJIT_NOTHROW { _hintValue = hintValue; }

  VarData* _vdata;
  uint32_t _hintId;
  uint32_t _hintValue;
};

// ============================================================================
// [AsmJit::EInstruction]
// ============================================================================

//! @brief Emittable that represents single instruction and its operands.
struct ASMJIT_API EInstruction : public Emittable
{
  // --------------------------------------------------------------------------
  // [Construction / Destruction]
  // --------------------------------------------------------------------------

  //! @brief Create a new @ref EInstruction instance.
  EInstruction(Compiler* c, uint32_t code, Operand* operandsData, uint32_t operandsCount) ASMJIT_NOTHROW;
  //! @brief Destroy the @ref EInstruction instance.
  virtual ~EInstruction() ASMJIT_NOTHROW;

  // --------------------------------------------------------------------------
  // [Emit]
  // --------------------------------------------------------------------------

  virtual void prepare(CompilerContext& cc) ASMJIT_NOTHROW;
  virtual Emittable* translate(CompilerContext& cc) ASMJIT_NOTHROW;

  virtual void emit(Assembler& a) ASMJIT_NOTHROW;

  // --------------------------------------------------------------------------
  // [Utilities]
  // --------------------------------------------------------------------------

  virtual int getMaxSize() const ASMJIT_NOTHROW;
  virtual bool _tryUnuseVar(VarData* v) ASMJIT_NOTHROW;

  // --------------------------------------------------------------------------
  // [Instruction Code]
  // --------------------------------------------------------------------------

  //! @brief Get whether the instruction is special.
  inline bool isSpecial() const ASMJIT_NOTHROW { return _isSpecial; }

  //! @brief Get whether the instruction is FPU.
  inline bool isFPU() const ASMJIT_NOTHROW { return _isFPU; }

  //! @brief Get instruction code, see @c INST_CODE.
  inline uint32_t getCode() const ASMJIT_NOTHROW { return _code; }

  //! @brief Set instruction code to @a code.
  //!
  //! Please do not modify instruction code if you are not know what you are
  //! doing. Incorrect instruction code or operands can raise assertion() at
  //! runtime.
  inline void setCode(uint32_t code) ASMJIT_NOTHROW { _code = code; }

  // --------------------------------------------------------------------------
  // [Operands]
  // --------------------------------------------------------------------------

  //! @brief Get count of operands in operands array (number between 0 to 2 inclusive).
  inline uint32_t getOperandsCount() const ASMJIT_NOTHROW { return _operandsCount; }

  //! @brief Get operands array (3 operands total).
  inline Operand* getOperands() ASMJIT_NOTHROW { return _operands; }
  //! @brief Get operands array (3 operands total).
  inline const Operand* getOperands() const ASMJIT_NOTHROW { return _operands; }

  //! @brief Get memory operand.
  inline Mem* getMemOp() ASMJIT_NOTHROW { return _memOp; }
  //! @brief Set memory operand.
  inline void setMemOp(Mem* op) ASMJIT_NOTHROW { _memOp = op; }

  // --------------------------------------------------------------------------
  // [Variables]
  // --------------------------------------------------------------------------

  //! @brief Get count of variables in instruction operands (and in variables array).
  inline uint32_t getVariablesCount() const ASMJIT_NOTHROW { return _variablesCount; }

  //! @brief Get operands array (3 operands total).
  inline VarAllocRecord* getVariables() ASMJIT_NOTHROW { return _variables; }
  //! @brief Get operands array (3 operands total).
  inline const VarAllocRecord* getVariables() const ASMJIT_NOTHROW { return _variables; }

  // --------------------------------------------------------------------------
  // [Jump]
  // --------------------------------------------------------------------------

  //! @brief Get possible jump target.
  //!
  //! If this instruction is conditional or normal jump then return value is
  //! label location (ETarget instance), otherwise return value is @c NULL.
  virtual ETarget* getJumpTarget() const ASMJIT_NOTHROW;

  // --------------------------------------------------------------------------
  // [Members]
  // --------------------------------------------------------------------------

protected:
  //! @brief Instruction code, see @c INST_CODE.
  uint32_t _code;

  //! @brief Emit options, see @c EMIT_OPTIONS.
  uint32_t _emitOptions;

  //! @brief Operands count.
  uint32_t _operandsCount;

  //! @brief Variables count.
  uint32_t _variablesCount;

  //! @brief Operands.
  Operand* _operands;
  //! @brief Memory operand (if instruction contains any).
  Mem* _memOp;

  //! @brief Variables (extracted from operands).
  VarAllocRecord* _variables;

  //! @brief Whether the instruction is special.
  bool _isSpecial;
  //! @brief Whether the instruction is FPU.
  bool _isFPU;

  //! @brief Whether the one of the operands is GPB.Lo register.
  bool _isGPBLoUsed;
  //! @brief Whether the one of the operands is GPB.Hi register.
  bool _isGPBHiUsed;

  friend struct EFunction;
  friend struct CompilerContext;
  friend struct CompilerCore;

private:
  ASMJIT_DISABLE_COPY(EInstruction)
};

// ============================================================================
// [AsmJit::EJmp]
// ============================================================================

//! @brief Emittable that represents single instruction that can jump somewhere.
struct ASMJIT_API EJmp : public EInstruction
{
  // --------------------------------------------------------------------------
  // [Construction / Destruction]
  // --------------------------------------------------------------------------

  EJmp(Compiler* c, uint32_t code, Operand* operandsData, uint32_t operandsCount) ASMJIT_NOTHROW;
  virtual ~EJmp() ASMJIT_NOTHROW;

  // --------------------------------------------------------------------------
  // [Emit]
  // --------------------------------------------------------------------------

  virtual void prepare(CompilerContext& cc) ASMJIT_NOTHROW;
  virtual Emittable* translate(CompilerContext& cc) ASMJIT_NOTHROW;
  virtual void emit(Assembler& a) ASMJIT_NOTHROW;

  void _doJump(CompilerContext& cc) ASMJIT_NOTHROW;

  // --------------------------------------------------------------------------
  // [Jump]
  // --------------------------------------------------------------------------

  virtual ETarget* getJumpTarget() const ASMJIT_NOTHROW;

  inline EJmp* getJumpNext() const ASMJIT_NOTHROW { return _jumpNext; }
  inline bool isTaken() const ASMJIT_NOTHROW { return _isTaken; }

  // --------------------------------------------------------------------------
  // [Members]
  // --------------------------------------------------------------------------

protected:
  ETarget* _jumpTarget;
  EJmp *_jumpNext;
  StateData* _state;
  bool _isTaken;

  friend struct EFunction;
  friend struct CompilerContext;
  friend struct CompilerCore;

private:
  ASMJIT_DISABLE_COPY(EJmp)
};

// ============================================================================
// [AsmJit::EFunction]
// ============================================================================

//! @brief Function emittable used to generate C/C++ functions.
//!
//! Functions are base blocks for generating assembler output. Each generated
//! assembler stream needs standard entry and leave sequences thats compatible
//! to the operating system conventions - Application Binary Interface (ABI).
//!
//! Function class can be used to generate entry (prolog) and leave (epilog)
//! sequences that is compatible to a given calling convention and to allocate
//! and manage variables that can be allocated to registers or spilled.
//!
//! @note To create function use @c AsmJit::Compiler::newFunction() method, do
//! not create @c EFunction instances using other ways.
//!
//! @sa @c State, @c Var.
struct ASMJIT_API EFunction : public Emittable
{
  // --------------------------------------------------------------------------
  // [Construction / Destruction]
  // --------------------------------------------------------------------------

  //! @brief Create new @c Function instance.
  //!
  //! @note Always use @c AsmJit::Compiler::newFunction() to create @c Function
  //! instance.
  EFunction(Compiler* c) ASMJIT_NOTHROW;
  //! @brief Destroy @c Function instance.
  virtual ~EFunction() ASMJIT_NOTHROW;

  // --------------------------------------------------------------------------
  // [Emit]
  // --------------------------------------------------------------------------

  virtual void prepare(CompilerContext& cc) ASMJIT_NOTHROW;

  // --------------------------------------------------------------------------
  // [Utilities]
  // --------------------------------------------------------------------------

  virtual int getMaxSize() const ASMJIT_NOTHROW;

  // --------------------------------------------------------------------------
  // [Function Prototype (Calling Convention + Arguments) / Return Value]
  // --------------------------------------------------------------------------

  inline const FunctionPrototype& getPrototype() const ASMJIT_NOTHROW { return _functionPrototype; }
  inline uint32_t getHint(uint32_t hint) ASMJIT_NOTHROW { return _hints[hint]; }

  void setPrototype(
    uint32_t callingConvention, 
    const uint32_t* arguments, 
    uint32_t argumentsCount,
    uint32_t returnValue) ASMJIT_NOTHROW;
  void setHint(uint32_t hint, uint32_t value) ASMJIT_NOTHROW;

  inline EProlog* getProlog() const ASMJIT_NOTHROW { return _prolog; }
  inline EEpilog* getEpilog() const ASMJIT_NOTHROW { return _epilog; }

  inline EFunctionEnd* getEnd() const ASMJIT_NOTHROW { return _end; }

  //! @brief Create variables from FunctionPrototype declaration. This is just
  //! parsing what FunctionPrototype generated for current function calling
  //! convention and arguments.
  void _createVariables() ASMJIT_NOTHROW;

  //! @brief Prepare variables (ids, names, scope, registers).
  void _prepareVariables(Emittable* first) ASMJIT_NOTHROW;

  //! @brief Allocate variables (setting correct state, changing masks, etc).
  void _allocVariables(CompilerContext& cc) ASMJIT_NOTHROW;

  void _preparePrologEpilog(CompilerContext& cc) ASMJIT_NOTHROW;
  void _dumpFunction(CompilerContext& cc) ASMJIT_NOTHROW;
  void _emitProlog(CompilerContext& cc) ASMJIT_NOTHROW;
  void _emitEpilog(CompilerContext& cc) ASMJIT_NOTHROW;

  // --------------------------------------------------------------------------
  // [Function-Call]
  // --------------------------------------------------------------------------

  //! @brief Reserve stack for calling other function and mark function as
  //! callee.
  void reserveStackForFunctionCall(int32_t size);

  // --------------------------------------------------------------------------
  // [Labels]
  // --------------------------------------------------------------------------

  //! @brief Get function entry label.
  //!
  //! Entry label can be used to call this function from another code that's
  //! being generated.
  inline const Label& getEntryLabel() const ASMJIT_NOTHROW { return _entryLabel; }

  //! @brief Get function exit label.
  //!
  //! Use exit label to jump to function epilog.
  inline const Label& getExitLabel() const ASMJIT_NOTHROW { return _exitLabel; }

  // --------------------------------------------------------------------------
  // [Misc]
  // --------------------------------------------------------------------------

  //! @brief Set the _isEspAdjusted member to true.
  //!
  //! This method is used to tell compiler that the ESP/RSP must be adjusted in
  //! function prolog/epilog, because the stack is manipulated (usually caused
  //! by the function call, see @c ECall).
  inline void mustAdjustEsp() { _isEspAdjusted = true; }

  // --------------------------------------------------------------------------
  // [Members]
  // --------------------------------------------------------------------------

protected:
  //! @brief Function prototype.
  FunctionPrototype _functionPrototype;
  //! @brief Function arguments (variable IDs).
  VarData** _argumentVariables;
  //! @brief Function hints.
  uint32_t _hints[16];

  //! @brief Whether the function stack is aligned by 16-bytes by OS.
  //!
  //! This is always true for 64-bit mode and for linux.
  bool _isStackAlignedByOsTo16Bytes;

  //! @brief Whether the function stack (for variables) is aligned manually
  //! by function to 16-bytes.
  //!
  //! This makes sense only if _isStackAlignedByOsTo16Bytes is false and MOVDQA
  //! instruction or other SSE/SSE2 instructions are used to work with variable
  //! stored on the stack.
  //!
  //! Value is determined automatically by these factors, expectations are:
  //!
  //!   1. There is 16-byte wide variable which address was used (alloc, spill,
  //!      op).
  //!   2. Function can't be naked.
  bool _isStackAlignedByFnTo16Bytes;

  //! @brief Whether the function is using naked prolog / epilog
  //!
  //! Naked prolog / epilog means to omit saving and restoring EBP.
  bool _isNaked;

  //! @brief Whether the ESP register is adjusted by the stack size needed
  //! to save registers and function variables.
  //!
  //! Esp is adjusted by 'sub' instruction in prolog and by add function in
  //! epilog (only if function is not naked).
  bool _isEspAdjusted;

  //! @brief Whether another function is called from this function.
  //!
  //! If another function is called from this function, it's needed to prepare
  //! stack for it. If this member is true then it's likely that true will be
  //! also @c _isEspAdjusted one.
  bool _isCaller;

  //! @brief Whether to emit prolog / epilog sequence using push & pop
  //! instructions (the default).
  bool _pePushPop;

  //! @brief Whether to emit EMMS instruction in epilog (auto-detected).
  bool _emitEMMS;

  //! @brief Whether to emit SFence instruction in epilog (auto-detected).
  //!
  //! @note Combination of @c _emitSFence and @c _emitLFence will result in
  //! emitting mfence.
  bool _emitSFence;

  //! @brief Whether to emit LFence instruction in epilog (auto-detected).
  //!
  //! @note Combination of @c _emitSFence and @c _emitLFence will result in
  //! emitting mfence.
  bool _emitLFence;

  //! @brief Whether the function is finished using @c Compiler::endFunction().
  bool _finished;

  //! @brief Bitfield containing modified and preserved GP registers.
  uint32_t _modifiedAndPreservedGP;

  //! @brief Bitfield containing modified and preserved MM registers.
  uint32_t _modifiedAndPreservedMM;

  //! @brief Bitfield containing modified and preserved XMM registers.
  uint32_t _modifiedAndPreservedXMM;

  //! @brief ID mov movdqa instruction (@c INST_MOVDQA or @c INST_MOVDQU).
  //!
  //! The value is based on stack alignment. If it's guaranteed that stack
  //! is aligned to 16-bytes then @c INST_MOVDQA instruction is used, otherwise
  //! the @c INST_MOVDQU instruction is used for 16-byte mov.
  uint32_t _movDqaInstruction;

  //! @brief Prolog / epilog stack size for PUSH/POP sequences.
  int32_t _pePushPopStackSize;
  //! @brief Prolog / epilog stack size for MOV sequences.
  int32_t _peMovStackSize;
  //! @brief Prolog / epilog stack adjust size (to make it 16-byte aligned).
  int32_t _peAdjustStackSize;

  //! @brief Memory stack size (for all variables and temporary memory).
  int32_t _memStackSize;
  //! @brief Like @c _memStackSize, but aligned to 16-bytes.
  int32_t _memStackSize16;

  //! @brief Stack size needed to call other functions.
  int32_t _functionCallStackSize;

  //! @brief Function entry label.
  Label _entryLabel;
  //! @brief Function exit label.
  Label _exitLabel;

  //! @brief Function prolog emittable.
  EProlog* _prolog;
  //! @brief Function epilog emittable.
  EEpilog* _epilog;
  //! @brief Dummy emittable, signalizes end of function.
  EFunctionEnd* _end;

private:
  friend struct CompilerContext;
  friend struct CompilerCore;
  friend struct EProlog;
  friend struct EEpilog;
};

// ============================================================================
// [AsmJit::EProlog]
// ============================================================================

//! @brief Prolog emittable.
struct ASMJIT_API EProlog : public Emittable
{
  // --------------------------------------------------------------------------
  // [Construction / Destruction]
  // --------------------------------------------------------------------------

  //! @brief Create a new @ref EProlog instance.
  EProlog(Compiler* c, EFunction* f) ASMJIT_NOTHROW;
  //! @brief Destroy the @ref EProlog instance.
  virtual ~EProlog() ASMJIT_NOTHROW;

  // --------------------------------------------------------------------------
  // [Emit]
  // --------------------------------------------------------------------------

  virtual void prepare(CompilerContext& cc) ASMJIT_NOTHROW;
  virtual Emittable* translate(CompilerContext& cc) ASMJIT_NOTHROW;

  // --------------------------------------------------------------------------
  // [Methods]
  // --------------------------------------------------------------------------

  //! @brief Get function associated with this prolog.
  inline EFunction* getFunction() const ASMJIT_NOTHROW { return _function; }

  // --------------------------------------------------------------------------
  // [Members]
  // --------------------------------------------------------------------------

protected:
  //! @brief Prolog owner function.
  EFunction* _function;

private:
  friend struct CompilerCore;
  friend struct EFunction;
};

// ============================================================================
// [AsmJit::EEpilog]
// ============================================================================

//! @brief Epilog emittable.
struct ASMJIT_API EEpilog : public Emittable
{
  // --------------------------------------------------------------------------
  // [Construction / Destruction]
  // --------------------------------------------------------------------------

  //! @brief Create a new @ref EEpilog instance.
  EEpilog(Compiler* c, EFunction* f) ASMJIT_NOTHROW;
  //! @brief Destroy the @ref EProlog instance.
  virtual ~EEpilog() ASMJIT_NOTHROW;

  // --------------------------------------------------------------------------
  // [Emit]
  // --------------------------------------------------------------------------

  virtual void prepare(CompilerContext& cc) ASMJIT_NOTHROW;
  virtual Emittable* translate(CompilerContext& cc) ASMJIT_NOTHROW;

  // --------------------------------------------------------------------------
  // [Methods]
  // --------------------------------------------------------------------------

  //! @brief Get function associated with this epilog.
  inline EFunction* getFunction() const ASMJIT_NOTHROW { return _function; }

  // --------------------------------------------------------------------------
  // [Members]
  // --------------------------------------------------------------------------

protected:
  //! @brief Epilog owner function.
  EFunction* _function;

private:
  friend struct CompilerCore;
  friend struct EFunction;
};

// ============================================================================
// [AsmJit::ECall]
// ============================================================================

//! @brief Function call.
struct ASMJIT_API ECall : public Emittable
{
  // --------------------------------------------------------------------------
  // [Construction / Destruction]
  // --------------------------------------------------------------------------

  //! @brief Create a new @ref ECall instance.
  ECall(Compiler* c, EFunction* caller, const Operand* target) ASMJIT_NOTHROW;
  //! @brief Destroy the @ref ECall instance.
  virtual ~ECall() ASMJIT_NOTHROW;

  // --------------------------------------------------------------------------
  // [Emit]
  // --------------------------------------------------------------------------

  virtual void prepare(CompilerContext& cc) ASMJIT_NOTHROW;
  virtual Emittable* translate(CompilerContext& cc) ASMJIT_NOTHROW;

  // --------------------------------------------------------------------------
  // [Utilities]
  // --------------------------------------------------------------------------

  virtual int getMaxSize() const ASMJIT_NOTHROW;
  virtual bool _tryUnuseVar(VarData* v) ASMJIT_NOTHROW;

  // --------------------------------------------------------------------------
  // [Internal]
  // --------------------------------------------------------------------------

protected:

  uint32_t _findTemporaryGpRegister(CompilerContext& cc) ASMJIT_NOTHROW;
  uint32_t _findTemporaryXmmRegister(CompilerContext& cc) ASMJIT_NOTHROW;

  VarData* _getOverlappingVariable(CompilerContext& cc,
    const FunctionPrototype::Argument& argType) const ASMJIT_NOTHROW;

  void _moveAllocatedVariableToStack(CompilerContext& cc,
    VarData* vdata, const FunctionPrototype::Argument& argType) ASMJIT_NOTHROW;

  void _moveSpilledVariableToStack(CompilerContext& cc,
    VarData* vdata, const FunctionPrototype::Argument& argType,
    uint32_t temporaryGpReg,
    uint32_t temporaryXmmReg) ASMJIT_NOTHROW;

  void _moveSrcVariableToRegister(CompilerContext& cc,
    VarData* vdata, const FunctionPrototype::Argument& argType) ASMJIT_NOTHROW;

  // --------------------------------------------------------------------------
  // [Function Prototype (Calling Convention + Arguments) / Return Value]
  // --------------------------------------------------------------------------

public:

  //! @brief Get function prototype.
  inline const FunctionPrototype& getPrototype() const ASMJIT_NOTHROW { return _functionPrototype; }

  //! @brief Set function prototype.
  inline void setPrototype(uint32_t cconv, const FunctionDefinition& def) ASMJIT_NOTHROW
  {
    _setPrototype(
      cconv,
      def.getArguments(), 
      def.getArgumentsCount(), 
      def.getReturnValue());
  }

  //! @brief Set function prototype (internal).
  void _setPrototype(
    uint32_t callingConvention,
    const uint32_t* arguments,
    uint32_t argumentsCount,
    uint32_t returnValue) ASMJIT_NOTHROW;

  //! @brief Set function argument @a i to @a var.
  bool setArgument(uint32_t i, const BaseVar& var) ASMJIT_NOTHROW;
  //! @brief Set function argument @a i to @a imm.
  bool setArgument(uint32_t i, const Imm& imm) ASMJIT_NOTHROW;

  //! @brief Set return value to 
  bool setReturn(const Operand& first, const Operand& second = Operand()) ASMJIT_NOTHROW;

  // --------------------------------------------------------------------------
  // [Methods]
  // --------------------------------------------------------------------------

  //! @brief Get caller.
  inline EFunction* getCaller() const ASMJIT_NOTHROW { return _caller; }

  //! @brief Get operand (function address).
  inline Operand& getTarget() ASMJIT_NOTHROW { return _target; }
  //! @overload
  inline const Operand& getTarget() const ASMJIT_NOTHROW { return _target; }

  // --------------------------------------------------------------------------
  // [Members]
  // --------------------------------------------------------------------------

protected:
  //! @brief Function prototype.
  FunctionPrototype _functionPrototype;

  //! @brief Callee (the function that calls me).
  EFunction* _caller;

  //! @brief Arguments (operands).
  Operand* _args;

  //! @brief Operand (address of function, register, label, ...)
  Operand _target;

  //! @brief Return value (operands)
  Operand _ret[2];

  //! @brief Mask of GP registers used as function arguments.
  uint32_t _gpParams;
  //! @brief Mask of MM registers used as function arguments.
  uint32_t _mmParams;
  //! @brief Mask of XMM registers used as function arguments.
  uint32_t _xmmParams;

  //! @brief Variables count.
  uint32_t _variablesCount;

  //! @brief Variables (extracted from operands).
  VarCallRecord* _variables;
  //! @brief Argument index to @c VarCallRecord.
  VarCallRecord* _argumentToVarRecord[FUNC_MAX_ARGS];

private:
  friend struct CompilerCore;
};

// ============================================================================
// [AsmJit::ERet]
// ============================================================================

//! @brief Function return.
struct ASMJIT_API ERet : public Emittable
{
  // --------------------------------------------------------------------------
  // [Construction / Destruction]
  // --------------------------------------------------------------------------

  //! @brief Create a new @ref ERet instance.
  ERet(Compiler* c, EFunction* function, const Operand* first, const Operand* second) ASMJIT_NOTHROW;
  //! @brief Destroy the @ref ERet instance.
  virtual ~ERet() ASMJIT_NOTHROW;

  // --------------------------------------------------------------------------
  // [Emit]
  // --------------------------------------------------------------------------

  virtual void prepare(CompilerContext& cc) ASMJIT_NOTHROW;
  virtual Emittable* translate(CompilerContext& cc) ASMJIT_NOTHROW;
  virtual void emit(Assembler& a) ASMJIT_NOTHROW;

  // --------------------------------------------------------------------------
  // [Utilities]
  // --------------------------------------------------------------------------

  virtual int getMaxSize() const ASMJIT_NOTHROW;

  // --------------------------------------------------------------------------
  // [Methods]
  // --------------------------------------------------------------------------

  //! @Brief Get function.
  inline EFunction* getFunction() ASMJIT_NOTHROW { return _function; }

  //! @brief Get operand (function address).
  inline Operand& getFirst() ASMJIT_NOTHROW { return _ret[0]; }
  //! @brief Get operand (function address).
  inline Operand& getSecond() ASMJIT_NOTHROW { return _ret[1]; }
  //! @overload
  inline const Operand& getFirst() const ASMJIT_NOTHROW { return _ret[0]; }
  //! @overload
  inline const Operand& getSecond() const ASMJIT_NOTHROW { return _ret[1]; }

  //! @brief Get whether jump to epilog have to be emitted.
  bool shouldEmitJumpToEpilog() const ASMJIT_NOTHROW;

  // --------------------------------------------------------------------------
  // [Members]
  // --------------------------------------------------------------------------

protected:
  //! @brief Function.
  EFunction* _function;
  //! @brief Return value (operands)
  Operand _ret[2];

private:
  friend struct CompilerCore;
};

// ============================================================================
// [AsmJit::CompilerContext]
// ============================================================================

//! @internal
//!
//! @brief Compiler context is used by @ref Compiler.
//!
//! Compiler context is used during compilation and normally developer doesn't
//! need access to it. The context is user per function (it's reset after each
//! function is generated).
struct ASMJIT_API CompilerContext
{
  // --------------------------------------------------------------------------
  // [Construction / Destruction]
  // --------------------------------------------------------------------------

  //! @brief Create a new @ref CompilerContext instanc…