// AsmJit - Complete JIT Assembler for C++ Language.

// Copyright (c) 2008-2010, Petr Kobalicek <kobalicek.petr@gmail.com>
//
// Permission is hereby granted, free of charge, to any person
// obtaining a copy of this software and associated documentation
// files (the "Software"), to deal in the Software without
// restriction, including without limitation the rights to use,
// copy, modify, merge, publish, distribute, sublicense, and/or sell
// copies of the Software, and to permit persons to whom the
// Software is furnished to do so, subject to the following
// conditions:
// 
// The above copyright notice and this permission notice shall be
// included in all copies or substantial portions of the Software.
// 
// THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND,
// EXPRESS OR IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES
// OF MERCHANTABILITY, FITNESS FOR A PARTICULAR PURPOSE AND
// NONINFRINGEMENT. IN NO EVENT SHALL THE AUTHORS OR COPYRIGHT
// HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER LIABILITY,
// WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING
// FROM, OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR
// OTHER DEALINGS IN THE SOFTWARE.

// We are using sprintf() here.
#if defined(_MSC_VER) && !defined(_CRT_SECURE_NO_WARNINGS)
#define _CRT_SECURE_NO_WARNINGS
#endif // _MSC_VER

// [Dependencies]
#include "Assembler.h"
#include "CodeGenerator.h"
#include "CpuInfo.h"
#include "Defs.h"
#include "Logger.h"
#include "MemoryManager.h"
#include "Platform.h"
#include "Util_p.h"

#include <stdarg.h>
#include <stdio.h>
#include <stdlib.h>

// A little bit C++.
#include <new>

// [Api-Begin]
#include "ApiBegin.h"

namespace AsmJit {

#if defined(ASMJIT_X64)

// ============================================================================
// [AsmJit::TrampolineWriter]
// ============================================================================

//! @brief Class used to determine size of trampoline and as trampoline writer.
struct ASMJIT_HIDDEN TrampolineWriter
{
  // Size of trampoline
  enum {
    TRAMPOLINE_JMP = 6,
    TRAMPOLINE_ADDR = sizeof(sysint_t),

    TRAMPOLINE_SIZE = TRAMPOLINE_JMP + TRAMPOLINE_ADDR
  };

  // Write trampoline into code at address @a code that will jump to @a target.
  static void writeTrampoline(uint8_t* code, void* target)
  {
    // Jmp.
    code[0] = 0xFF;
    // ModM (RIP addressing).
    code[1] = 0x25;
    // Offset (zero).
    ((uint32_t*)(code + 2))[0] = 0;
    // Absolute address.
    ((sysuint_t*)(code + TRAMPOLINE_JMP))[0] = (sysuint_t)target;
  }
};

#endif // ASMJIT_X64

// ============================================================================
// [AsmJit::AssemblerCore - Construction / Destruction]
// ============================================================================

AssemblerCore::AssemblerCore(CodeGenerator* codeGenerator) ASMJIT_NOTHROW :
  _codeGenerator(codeGenerator != NULL ? codeGenerator : CodeGenerator::getGlobal()),
  _zone(16384 - sizeof(Zone::Chunk) - 32),
  _logger(NULL),
  _error(0),
  _properties((1 << PROPERTY_OPTIMIZE_ALIGN)),
  _emitOptions(0),
  _buffer(32), // Max instruction length is 15, but we can align up to 32 bytes.
  _trampolineSize(0),
  _unusedLinks(NULL),
  _comment(NULL)
{
}

AssemblerCore::~AssemblerCore() ASMJIT_NOTHROW
{
}

// ============================================================================
// [AsmJit::AssemblerCore - Logging]
// ============================================================================

void AssemblerCore::setLogger(Logger* logger) ASMJIT_NOTHROW
{
  _logger = logger;
}

// ============================================================================
// [AsmJit::AssemblerCore - Error Handling]
// ============================================================================

void AssemblerCore::setError(uint32_t error) ASMJIT_NOTHROW
{
  _error = error;
  if (_error == ERROR_NONE) return;

  if (_logger)
  {
    _logger->logFormat("*** ASSEMBLER ERROR: %s (%u).\n",
      getErrorString(error),
      (unsigned int)error);
  }
}

// ============================================================================
// [AsmJit::AssemblerCore - Properties]
// ============================================================================

uint32_t AssemblerCore::getProperty(uint32_t propertyId)
{
  return (_properties & (1 << propertyId)) != 0;
}

void AssemblerCore::setProperty(uint32_t propertyId, uint32_t value)
{
  if (value)
    _properties |= (1 << propertyId);
  else
    _properties &= ~(1 << propertyId);
}

// ============================================================================
// [AsmJit::AssemblerCore - Buffer]
// ============================================================================

void AssemblerCore::clear() ASMJIT_NOTHROW
{
  _buffer.clear();
  _relocData.clear();
  _zone.clear();

  if (_error) setError(ERROR_NONE);
}

void AssemblerCore::free() ASMJIT_NOTHROW
{
  _zone.freeAll();
  _buffer.free();
  _relocData.free();

  if (_error) setError(ERROR_NONE);
}

uint8_t* AssemblerCore::takeCode() ASMJIT_NOTHROW
{
  uint8_t* code = _buffer.take();
  _relocData.clear();
  _zone.clear();

  if (_error) setError(ERROR_NONE);
  return code;
}

// ============================================================================
// [AsmJit::AssemblerCore - Stream Setters / Getters]
// ============================================================================

void AssemblerCore::setVarAt(sysint_t pos, sysint_t i, uint8_t isUnsigned, uint32_t size) ASMJIT_NOTHROW
{
  if (size == 1 && !isUnsigned) setByteAt (pos, (int8_t  )i);
  else if (size == 1 &&  isUnsigned) setByteAt (pos, (uint8_t )i);
  else if (size == 2 && !isUnsigned) setWordAt (pos, (int16_t )i);
  else if (size == 2 &&  isUnsigned) setWordAt (pos, (uint16_t)i);
  else if (size == 4 && !isUnsigned) setDWordAt(pos, (int32_t )i);
  else if (size == 4 &&  isUnsigned) setDWordAt(pos, (uint32_t)i);
#if defined(ASMJIT_X64)
  else if (size == 8 && !isUnsigned) setQWordAt(pos, (int64_t )i);
  else if (size == 8 &&  isUnsigned) setQWordAt(pos, (uint64_t)i);
#endif // ASMJIT_X64
  else
    ASMJIT_ASSERT(0);
}

// ============================================================================
// [AsmJit::AssemblerCore - Assembler Emitters]
// ============================================================================

bool AssemblerCore::canEmit() ASMJIT_NOTHROW
{
  // If there is an error, we can't emit another instruction until last error
  // is cleared by calling @c setError(ERROR_NONE). If something caused an error
  // while generating code it's probably fatal in all cases. You can't use 
  // generated code, because you are not sure about its status.
  if (_error) return false;

  // The ensureSpace() method returns true on success and false on failure. We
  // are catching return value and setting error code here.
  if (ensureSpace()) return true;

  // If we are here, there is memory allocation error. Note that this is HEAP
  // allocation error, virtual allocation error can be caused only by
  // AsmJit::VirtualMemory class!
  setError(ERROR_NO_HEAP_MEMORY);
  return false;
}

void AssemblerCore::_emitSegmentPrefix(const Operand& rm) ASMJIT_NOTHROW
{
  static const uint8_t segmentPrefixCode[6] =
  {
    0x26, // ES
    0x2E, // SS
    0x36, // SS
    0x3E, // DS
    0x64, // FS
    0x65  // GS
  };

  uint32_t segmentPrefix;
  
  if (!rm.isMem())
    return;

  if ((segmentPrefix = reinterpret_cast<const Mem&>(rm).getSegmentPrefix()) >= REG_NUM_SEGMENT)
    return;

  _emitByte(segmentPrefixCode[segmentPrefix]);
}

void AssemblerCore::_emitModM(
  uint8_t opReg, const Mem& mem, sysint_t immSize) ASMJIT_NOTHROW
{
  ASMJIT_ASSERT(mem.getType() == OPERAND_MEM);

  uint8_t baseReg = mem.getBase() & 0x7;
  uint8_t indexReg = mem.getIndex() & 0x7;
  sysint_t disp = mem.getDisplacement();
  uint32_t shift = mem.getShift();

  if (mem.getMemType() == OPERAND_MEM_NATIVE)
  {
    // [base + displacemnt]
    if (!mem.hasIndex())
    {
      // ESP/RSP/R12 == 4
      if (baseReg == 4)
      {
        uint8_t mod = 0;

        if (disp)
        {
          mod = Util::isInt8(disp) ? 1 : 2;
        }

        _emitMod(mod, opReg, 4);
        _emitSib(0, 4, 4);

        if (disp)
        {
          if (Util::isInt8(disp))
            _emitByte((int8_t)disp);
          else
            _emitInt32((int32_t)disp);
        }
      }
      // EBP/RBP/R13 == 5
      else if (baseReg != 5 && disp == 0)
      {
        _emitMod(0, opReg, baseReg);
      }
      else if (Util::isInt8(disp))
      {
        _emitMod(1, opReg, baseReg);
        _emitByte((int8_t)disp);
      }
      else
      {
        _emitMod(2, opReg, baseReg);
        _emitInt32((int32_t)disp);
      }
    }

    // [base + index * scale + displacemnt]
    else
    {
      // ASMJIT_ASSERT(indexReg != RID_ESP);

      // EBP/RBP/R13 == 5
      if (baseReg != 5 && disp == 0)
      {
        _emitMod(0, opReg, 4);
        _emitSib(shift, indexReg, baseReg);
      }
      else if (Util::isInt8(disp))
      {
        _emitMod(1, opReg, 4);
        _emitSib(shift, indexReg, baseReg);
        _emitByte((int8_t)disp);
      }
      else
      {
        _emitMod(2, opReg, 4);
        _emitSib(shift, indexReg, baseReg);
        _emitInt32((int32_t)disp);
      }
    }
  }

  // Address                       | 32-bit mode | 64-bit mode
  // ------------------------------+-------------+---------------
  // [displacement]                |   ABSOLUTE  | RELATIVE (RIP)
  // [index * scale + displacemnt] |   ABSOLUTE  | ABSOLUTE (ZERO EXTENDED)
  else
  {
    // - In 32-bit mode the absolute addressing model is used.
    // - In 64-bit mode the relative addressing model is used together with
    //   the absolute addressing. Main problem is that if instruction
    //   contains SIB then relative addressing (RIP) is not possible.

#if defined(ASMJIT_X86)

    if (mem.hasIndex())
    {
      // ASMJIT_ASSERT(mem.getMemIndex() != 4); // ESP/RSP == 4
      _emitMod(0, opReg, 4);
      _emitSib(shift, indexReg, 5);
    }
    else
    {
      _emitMod(0, opReg, 5);
    }

    // X86 uses absolute addressing model, all relative addresses will be
    // relocated to absolute ones.
    if (mem.getMemType() == OPERAND_MEM_LABEL)
    {
      LabelData& l_data = _labelData[mem._mem.base & OPERAND_ID_VALUE_MASK];
      RelocData r_data;
      uint32_t relocId = _relocData.getLength();

      // Relative addressing will be relocated to absolute address.
      r_data.type = RelocData::RELATIVE_TO_ABSOLUTE;
      r_data.size = 4;
      r_data.offset = getOffset();
      r_data.destination = disp;

      if (l_data.offset != -1)
      {
        // Bound label.
        r_data.destination += l_data.offset;

        // Add a dummy DWORD.
        _emitInt32(0);
      }
      else
      {
        // Non-bound label.
        _emitDisplacement(l_data, -4 - immSize, 4)->relocId = relocId;
      }

      _relocData.append(r_data);
    }
    else
    {
      // Absolute address
      _emitInt32( (int32_t)((uint8_t*)mem._mem.target + disp) );
    }

#else

    // X64 uses relative addressing model
    if (mem.getMemType() == OPERAND_MEM_LABEL)
    {
      LabelData& l_data = _labelData[mem._mem.base & OPERAND_ID_VALUE_MASK];

      if (mem.hasIndex())
      {
        // Indexing is not possible.
        setError(ERROR_ILLEGAL_ADDRESING);
        return;
      }

      // Relative address (RIP +/- displacement).
      _emitMod(0, opReg, 5);

      disp -= (4 + immSize);

      if (l_data.offset != -1)
      {
        // Bound label.
        disp += getOffset() - l_data.offset;

        // Add a dummy DWORD.
        _emitInt32((int32_t)disp);
      }
      else
      {
        // Non-bound label.
        _emitDisplacement(l_data, disp, 4);
      }
    }
    else
    {
      // Absolute address (truncated to 32-bits), this kind of address requires
      // SIB byte (4).
      _emitMod(0, opReg, 4);

      if (mem.hasIndex())
      {
        // ASMJIT_ASSERT(mem.getMemIndex() != 4); // ESP/RSP == 4
        _emitSib(shift, indexReg, 5);
      }
      else
      {
        _emitSib(0, 4, 5);
      }

      // Truncate to 32-bits.
      sysuint_t target = (sysuint_t)((uint8_t*)mem._mem.target + disp);

      if (target > (sysuint_t)0xFFFFFFFF)
      {
        if (_logger) 
        {
          _logger->logString("*** ASSEMBER WARNING - Absolute address truncated to 32-bits.\n");
        }
        target &= 0xFFFFFFFF;
      }

      _emitInt32( (int32_t)((uint32_t)target) );
    }

#endif // ASMJIT_X64

  }
}

void AssemblerCore::_emitModRM(
  uint8_t opReg, const Operand& op, sysint_t immSize) ASMJIT_NOTHROW
{
  ASMJIT_ASSERT(op.getType() == OPERAND_REG || op.getType() == OPERAND_MEM);

  if (op.getType() == OPERAND_REG)
    _emitModR(opReg, reinterpret_cast<const BaseReg&>(op).getRegCode());
  else
    _emitModM(opReg, reinterpret_cast<const Mem&>(op), immSize);
}

void AssemblerCore::_emitX86Inl(
  uint32_t opCode, uint8_t i16bit, uint8_t rexw, uint8_t reg, bool forceRexPrefix) ASMJIT_NOTHROW
{
  // 16-bit prefix.
  if (i16bit) _emitByte(0x66);

  // Instruction prefix.
  if (opCode & 0xFF000000) _emitByte((uint8_t)((opCode & 0xFF000000) >> 24));

  // REX prefix.
#if defined(ASMJIT_X64)
  _emitRexR(rexw, 0, reg, forceRexPrefix);
#endif // ASMJIT_X64

  // Instruction opcodes.
  if (opCode & 0x00FF0000) _emitByte((uint8_t)((opCode & 0x00FF0000) >> 16));
  if (opCode & 0x0000FF00) _emitByte((uint8_t)((opCode & 0x0000FF00) >>  8));

  _emitByte((uint8_t)(opCode & 0x000000FF) + (reg & 0x7));
}

void AssemblerCore::_emitX86RM(
  uint32_t opCode, uint8_t i16bit, uint8_t rexw, uint8_t o,
  const Operand& op, sysint_t immSize, bool forceRexPrefix) ASMJIT_NOTHROW
{
  // 16-bit prefix.
  if (i16bit) _emitByte(0x66);

  // Segment prefix.
  _emitSegmentPrefix(op);

  // Instruction prefix.
  if (opCode & 0xFF000000) _emitByte((uint8_t)((opCode & 0xFF000000) >> 24));

  // REX prefix.
#if defined(ASMJIT_X64)
  _emitRexRM(rexw, o, op, forceRexPrefix);
#endif // ASMJIT_X64

  // Instruction opcodes.
  if (opCode & 0x00FF0000) _emitByte((uint8_t)((opCode & 0x00FF0000) >> 16));
  if (opCode & 0x0000FF00) _emitByte((uint8_t)((opCode & 0x0000FF00) >>  8));
  _emitByte((uint8_t)(opCode & 0x000000FF));

  // Mod R/M.
  _emitModRM(o, op, immSize);
}

void AssemblerCore::_emitFpu(uint32_t opCode) ASMJIT_NOTHROW
{
  _emitOpCode(opCode);
}

void AssemblerCore::_emitFpuSTI(uint32_t opCode, uint32_t sti) ASMJIT_NOTHROW
{
  // Illegal stack offset.
  ASMJIT_ASSERT(0 <= sti && sti < 8);
  _emitOpCode(opCode + sti);
}

void AssemblerCore::_emitFpuMEM(uint32_t opCode, uint8_t opReg, const Mem& mem) ASMJIT_NOTHROW
{
  // Segment prefix.
  _emitSegmentPrefix(mem);

  // Instruction prefix.
  if (opCode & 0xFF000000) _emitByte((uint8_t)((opCode & 0xFF000000) >> 24));

  // REX prefix.
#if defined(ASMJIT_X64)
  _emitRexRM(0, opReg, mem, false);
#endif // ASMJIT_X64

  // Instruction opcodes.
  if (opCode & 0x00FF0000) _emitByte((uint8_t)((opCode & 0x00FF0000) >> 16));
  if (opCode & 0x0000FF00) _emitByte((uint8_t)((opCode & 0x0000FF00) >>  8));

  _emitByte((uint8_t)((opCode & 0x000000FF)));
  _emitModM(opReg, mem, 0);
}

void AssemblerCore::_emitMmu(uint32_t opCode, uint8_t rexw, uint8_t opReg,
  const Operand& src, sysint_t immSize) ASMJIT_NOTHROW
{
  // Segment prefix.
  _emitSegmentPrefix(src);

  // Instruction prefix.
  if (opCode & 0xFF000000) _emitByte((uint8_t)((opCode & 0xFF000000) >> 24));

  // REX prefix.
#if defined(ASMJIT_X64)
  _emitRexRM(rexw, opReg, src, false);
#endif // ASMJIT_X64

  // Instruction opcodes.
  if (opCode & 0x00FF0000) _emitByte((uint8_t)((opCode & 0x00FF0000) >> 16));

  // No checking, MMX/SSE instructions have always two opcodes or more.
  _emitByte((uint8_t)((opCode & 0x0000FF00) >> 8));
  _emitByte((uint8_t)((opCode & 0x000000FF)));

  if (src.isReg())
    _emitModR(opReg, reinterpret_cast<const BaseReg&>(src).getRegCode());
  else
    _emitModM(opReg, reinterpret_cast<const Mem&>(src), immSize);
}

AssemblerCore::LabelLink* AssemblerCore::_emitDisplacement(
  LabelData& l_data, sysint_t inlinedDisplacement, int size) ASMJIT_NOTHROW
{
  ASMJIT_ASSERT(l_data.offset == -1);
  ASMJIT_ASSERT(size == 1 || size == 4);

  // Chain with label.
  LabelLink* link = _newLabelLink();
  link->prev = l_data.links;
  link->offset = getOffset();
  link->displacement = inlinedDisplacement;

  l_data.links = link;

  // Emit label size as dummy data.
  if (size == 1)
    _emitByte(0x01);
  else // if (size == 4)
    _emitDWord(0x04040404);

  return link;
}

void AssemblerCore::_emitJmpOrCallReloc(uint32_t instruction, void* target) ASMJIT_NOTHROW
{
  RelocData rd;

  rd.type = RelocData::ABSOLUTE_TO_RELATIVE_TRAMPOLINE;

#if defined(ASMJIT_X64)
  // If we are compiling in 64-bit mode, we can use trampoline if relative jump
  // is not possible.
  _trampolineSize += TrampolineWriter::TRAMPOLINE_SIZE;
#endif // ARCHITECTURE_SPECIFIC

  rd.size = 4;
  rd.offset = getOffset();
  rd.address = target;

  _relocData.append(rd);

  // Emit dummy 32-bit integer (will be overwritten by relocCode()).
  _emitInt32(0);
}

// Logging helpers.
static const char* operandSize[] =
{
  NULL,
  "byte ptr ",
  "word ptr ",
  NULL,
  "dword ptr ",
  NULL,
  NULL,
  NULL,
  "qword ptr ",
  NULL,
  "tword ptr ",
  NULL,
  NULL,
  NULL,
  NULL,
  NULL,
  "dqword ptr "
};

static const char segmentPrefixName[] =
  "es:\0"
  "cs:\0"
  "ss:\0"
  "ds:\0"
  "fs:\0"
  "gs:\0"
  "\0\0\0\0";

ASMJIT_HIDDEN char* dumpInstructionName(char* buf, uint32_t code) ASMJIT_NOTHROW
{
  ASMJIT_ASSERT(code < _INST_COUNT);
  return Util::mycpy(buf, instructionDescription[code].getName());
}

ASMJIT_HIDDEN char* dumpRegister(char* buf, uint32_t type, uint32_t index) ASMJIT_NOTHROW
{
  // NE == Not-Encodable.
  const char reg8l[] = "al\0\0" "cl\0\0" "dl\0\0" "bl\0\0" "spl\0"  "bpl\0"  "sil\0"  "dil\0" ;
  const char reg8h[] = "ah\0\0" "ch\0\0" "dh\0\0" "bh\0\0" "NE\0\0" "NE\0\0" "NE\0\0" "NE\0\0";
  const char reg16[] = "ax\0\0" "cx\0\0" "dx\0\0" "bx\0\0" "sp\0\0" "bp\0\0" "si\0\0" "di\0\0";

  switch (type)
  {
    case REG_TYPE_GPB_LO:
      if (index < 8)
        return Util::mycpy(buf, &reg8l[index*4]);

      *buf++ = 'r';
      goto _EmitID;

    case REG_TYPE_GPB_HI:
      if (index < 4)
        return Util::mycpy(buf, &reg8h[index*4]);

_EmitNE:
      return Util::mycpy(buf, "NE");

    case REG_TYPE_GPW:
      if (index < 8)
        return Util::mycpy(buf, &reg16[index*4]);

      *buf++ = 'r';
      buf = Util::myutoa(buf, index);
      *buf++ = 'w';
      return buf;

    case REG_TYPE_GPD:
      if (index < 8)
      {
        *buf++ = 'e';
        return Util::mycpy(buf, &reg16[index*4]);
      }

      *buf++ = 'r';
      buf = Util::myutoa(buf, index);
      *buf++ = 'd';
      return buf;
    
    case REG_TYPE_GPQ:
      *buf++ = 'r';

      if (index < 8)
        return Util::mycpy(buf, &reg16[index*4]);

_EmitID:
      return Util::myutoa(buf, index);
    
    case REG_TYPE_X87:
      *buf++ = 's';
      *buf++ = 't';
      goto _EmitID;
    
    case REG_TYPE_MM:
      *buf++ = 'm';
      *buf++ = 'm';
      goto _EmitID;
    
    case REG_TYPE_XMM:
      *buf++ = 'x';
      *buf++ = 'm';
      *buf++ = 'm';
      goto _EmitID;
    
    case REG_TYPE_YMM:
      *buf++ = 'y';
      *buf++ = 'm';
      *buf++ = 'm';
      goto _EmitID;

    case REG_TYPE_SEGMENT:
      if (index < REG_NUM_SEGMENT)
        return Util::mycpy(buf, &segmentPrefixName[index*4], 2);
      
      goto _EmitNE;

    default:
      return buf;
  }
}

ASMJIT_HIDDEN char* dumpOperand(char* buf, const Operand* op, uint32_t memRegType) ASMJIT_NOTHROW
{
  if (op->isReg())
  {
    const BaseReg& reg = reinterpret_cast<const BaseReg&>(*op);
    return dumpRegister(buf, reg.getRegType(), reg.getRegIndex());
  }
  else if (op->isMem())
  {
    const Mem& mem = reinterpret_cast<const Mem&>(*op);
    uint32_t segmentPrefix = mem.getSegmentPrefix();

    bool isAbsolute = false;

    if (op->getSize() <= 16)
    {
      buf = Util::mycpy(buf, operandSize[op->getSize()]);
    }

    if (segmentPrefix < REG_NUM_SEGMENT)
    {
      buf = Util::mycpy(buf, &segmentPrefixName[segmentPrefix * 4]);
    }

    *buf++ = '[';

    switch (mem.getMemType())
    {
      case OPERAND_MEM_NATIVE:
      {
        // [base + index*scale + displacement]
        buf = dumpRegister(buf, memRegType, mem.getBase());
        break;
      }
      case OPERAND_MEM_LABEL:
      {
        // [label + index*scale + displacement]
        buf += sprintf(buf, "L.%u", mem.getBase() & OPERAND_ID_VALUE_MASK);
        break;
      }
      case OPERAND_MEM_ABSOLUTE:
      {
        // [absolute]
        isAbsolute = true;
        buf = Util::myutoa(buf, (sysuint_t)mem.getTarget(), 16);
        break;
      }
    }

    if (mem.hasIndex())
    {
      buf = Util::mycpy(buf, " + ");
      buf = dumpRegister(buf, memRegType, mem.getIndex());

      if (mem.getShift())
      {
        buf = Util::mycpy(buf, " * ");
        *buf++ = "1248"[mem.getShift() & 3];
      }
    }

    if (mem.getDisplacement() && !isAbsolute)
    {
      sysint_t d = mem.getDisplacement();
      *buf++ = ' ';
      *buf++ = (d < 0) ? '-' : '+';
      *buf++ = ' ';
      buf = Util::myutoa(buf, d < 0 ? -d : d);
    }

    *buf++ = ']';
    return buf;
  }
  else if (op->isImm())
  {
    const Imm& i = reinterpret_cast<const Imm&>(*op);
    return Util::myitoa(buf, (sysint_t)i.getValue());
  }
  else if (op->isLabel())
  {
    return buf + sprintf(buf, "L.%u", op->getId() & OPERAND_ID_VALUE_MASK);
  }
  else
  {
    return Util::mycpy(buf, "None");
  }
}

static char* dumpInstruction(char* buf,
  uint32_t code,
  uint32_t emitOptions,
  const Operand* o0,
  const Operand* o1,
  const Operand* o2,
  uint32_t memRegType) ASMJIT_NOTHROW
{
  if (emitOptions & EMIT_OPTION_REX_PREFIX ) buf = Util::mycpy(buf, "rex ", 4);
  if (emitOptions & EMIT_OPTION_LOCK_PREFIX) buf = Util::mycpy(buf, "lock ", 5);
  if (emitOptions & EMIT_OPTION_SHORT_JUMP ) buf = Util::mycpy(buf, "short ", 6);

  // Dump instruction.
  buf = dumpInstructionName(buf, code);

  // Dump operands.
  if (!o0->isNone()) { *buf++ = ' '; buf = dumpOperand(buf, o0, memRegType); }
  if (!o1->isNone()) { *buf++ = ','; *buf++ = ' '; buf = dumpOperand(buf, o1, memRegType); }
  if (!o2->isNone()) { *buf++ = ','; *buf++ = ' '; buf = dumpOperand(buf, o2, memRegType); }

  return buf;
}

static char* dumpComment(char* buf, sysuint_t len, const uint8_t* binaryData, sysuint_t binaryLen, const char* comment)
{
  sysuint_t currentLength = len;
  sysuint_t commentLength = comment ? strlen(comment) : 0;

  if (binaryLen || commentLength)
  {
    sysuint_t align = 32;
    char sep = ';';

    // Truncate if comment is too long (it shouldn't be, larger than 80 seems to
    // be an exploit).
    if (commentLength > 80) commentLength = 80;

    for (sysuint_t i = (binaryLen == 0); i < 2; i++)
    {
      char* bufBegin = buf;

      // Append align.
      if (currentLength < align) 
      {
        buf = Util::myfill(buf, ' ', align - currentLength);
      }

      // Append separator.
      if (sep)
      {
        *buf++ = sep;
        *buf++ = ' ';
      }

      // Append binary data or comment.
      if (i == 0)
      {
        buf = Util::myhex(buf, binaryData, binaryLen);
        if (commentLength == 0) break;
      }
      else
      {
        buf = Util::mycpy(buf, comment, commentLength);
      }

      currentLength += (sysuint_t)(buf - bufBegin);
      align += 18;
      sep = '|';
    }
  }

  *buf++ = '\n';
  return buf;
}

// Used for NULL operands to translate them to OPERAND_NONE.
static const uint8_t _none[sizeof(Operand)] =
{
  0
};

static const Operand::RegData _patchedHiRegs[4] =
{// op         , size, { reserved0, reserved1 }, id           , code
  { OPERAND_REG, 1   , { 0        , 0         }, INVALID_VALUE, REG_TYPE_GPB_LO | 4 },
  { OPERAND_REG, 1   , { 0        , 0         }, INVALID_VALUE, REG_TYPE_GPB_LO | 5 },
  { OPERAND_REG, 1   , { 0        , 0         }, INVALID_VALUE, REG_TYPE_GPB_LO | 6 },
  { OPERAND_REG, 1   , { 0        , 0         }, INVALID_VALUE, REG_TYPE_GPB_LO | 7 }
};

void AssemblerCore::_emitInstruction(uint32_t code) ASMJIT_NOTHROW
{
  _emitInstruction(code, NULL, NULL, NULL);
}

void AssemblerCore::_emitInstruction(uint32_t code, const Operand* o0) ASMJIT_NOTHROW
{
  _emitInstruction(code, o0, NULL, NULL);
}

void AssemblerCore::_emitInstruction(uint32_t code, const Operand* o0, const Operand* o1) ASMJIT_NOTHROW
{
  _emitInstruction(code, o0, o1, NULL);
}

void AssemblerCore::_emitInstruction(uint32_t code, const Operand* o0, const Operand* o1, const Operand* o2) ASMJIT_NOTHROW
{
  const Operand* _loggerOperands[3];

  uint32_t bLoHiUsed = 0;
#if defined(ASMJIT_X86)
  uint32_t forceRexPrefix = false;
#else
  uint32_t forceRexPrefix = _emitOptions & EMIT_OPTION_REX_PREFIX;
#endif
  uint32_t memRegType = REG_TYPE_GPN;

#if defined(ASMJIT_DEBUG)
  bool assertIllegal = false;
#endif // ASMJIT_DEBUG

  const Imm* immOperand = NULL;
  uint32_t immSize;

#define _FINISHED() \
  goto end

#define _FINISHED_IMMEDIATE(_Operand_, _Size_) \
  do { \
    immOperand = reinterpret_cast<const Imm*>(_Operand_); \
    immSize = (_Size_); \
    goto emitImmediate; \
  } while (0)

  // Convert operands to OPERAND_NONE if needed.
  if (o0 == NULL) { o0 = reinterpret_cast<const Operand*>(_none); } else if (o0->isReg()) { bLoHiUsed |= o0->_reg.code & (REG_TYPE_GPB_LO | REG_TYPE_GPB_HI); }
  if (o1 == NULL) { o1 = reinterpret_cast<const Operand*>(_none); } else if (o1->isReg()) { bLoHiUsed |= o1->_reg.code & (REG_TYPE_GPB_LO | REG_TYPE_GPB_HI); }
  if (o2 == NULL) { o2 = reinterpret_cast<const Operand*>(_none); } else if (o2->isReg()) { bLoHiUsed |= o2->_reg.code & (REG_TYPE_GPB_LO | REG_TYPE_GPB_HI); }

  sysuint_t beginOffset = getOffset();
  const InstructionDescription* id = &instructionDescription[code];

  if (code >= _INST_COUNT)
  {
    setError(ERROR_UNKNOWN_INSTRUCTION);
    goto cleanup;
  }

  // Check if register operand is BPL, SPL, SIL, DIL and do action that depends
  // to current mode:
  //   - 64-bit: - Force REX prefix.
  //
  // Check if register operand is AH, BH, CH or DH and do action that depends
  // to current mode:
  //   - 32-bit: - Patch operand index (index += 4), because we are using
  //               different index what is used in opcode.
  //   - 64-bit: - Check whether there is REX prefix and raise error if it is.
  //             - Do the same as in 32-bit mode - patch register index.
  //
  // NOTE: This is a hit hacky, but I added this to older code-base and I have
  // no energy to rewrite it. Maybe in future all of this can be cleaned up!
  if (bLoHiUsed | forceRexPrefix)
  {
    _loggerOperands[0] = o0;
    _loggerOperands[1] = o1;
    _loggerOperands[2] = o2;

#if defined(ASMJIT_X64)
    // Check if there is register that makes this instruction un-encodable.

    forceRexPrefix |= (uint32_t)o0->isExtendedRegisterUsed();
    forceRexPrefix |= (uint32_t)o1->isExtendedRegisterUsed();
    forceRexPrefix |= (uint32_t)o2->isExtendedRegisterUsed();

    if      (o0->isRegType(REG_TYPE_GPB_LO) && (o0->_reg.code & REG_INDEX_MASK) >= 4) forceRexPrefix = true;
    else if (o1->isRegType(REG_TYPE_GPB_LO) && (o1->_reg.code & REG_INDEX_MASK) >= 4) forceRexPrefix = true;
    else if (o2->isRegType(REG_TYPE_GPB_LO) && (o2->_reg.code & REG_INDEX_MASK) >= 4) forceRexPrefix = true;

    if ((bLoHiUsed & REG_TYPE_GPB_HI) != 0 && forceRexPrefix)
    {
      goto illegalInstruction;
    }
#endif // ASMJIT_X64

    // Patch GPB.HI operand index.
    if ((bLoHiUsed & REG_TYPE_GPB_HI) != 0)
    {
      if (o0->isRegType(REG_TYPE_GPB_HI)) o0 = reinterpret_cast<const Operand*>(&_patchedHiRegs[o0->_reg.code & REG_INDEX_MASK]);
      if (o1->isRegType(REG_TYPE_GPB_HI)) o1 = reinterpret_cast<const Operand*>(&_patchedHiRegs[o1->_reg.code & REG_INDEX_MASK]);
      if (o2->isRegType(REG_TYPE_GPB_HI)) o2 = reinterpret_cast<const Operand*>(&_patchedHiRegs[o2->_reg.code & REG_INDEX_MASK]);
    }
  }

  // Check for buffer space (and grow if needed).
  if (!canEmit()) goto cleanup;

  if (_emitOptions & EMIT_OPTION_LOCK_PREFIX)
  {
    if (!id->isLockable()) goto illegalInstruction;
    _emitByte(0xF0);
  }

  switch (id->group)
  {
    case InstructionDescription::G_EMIT:
    {
      _emitOpCode(id->opCode[0]);
      _FINISHED();
    }

    case InstructionDescription::G_ALU:
    {
      uint32_t opCode = id->opCode[0];
      uint8_t opReg = (uint8_t)id->opCodeR;

      // Mem <- Reg
      if (o0->isMem() && o1->isReg())
      {
        _emitX86RM(opCode + (o1->getSize() != 1),
          o1->getSize() == 2,
          o1->getSize() == 8,
          reinterpret_cast<const GPReg&>(*o1).getRegCode(),
          reinterpret_cast<const Operand&>(*o0),
          0, forceRexPrefix);
        _FINISHED();
      }

      // Reg <- Reg|Mem
      if (o0->isReg() && o1->isRegMem())
      {
        _emitX86RM(opCode + 2 + (o0->getSize() != 1),
          o0->getSize() == 2,
          o0->getSize() == 8,
          reinterpret_cast<const GPReg&>(*o0).getRegCode(),
          reinterpret_cast<const Operand&>(*o1),
          0, forceRexPrefix);
        _FINISHED();
      }

      // AL, AX, EAX, RAX register shortcuts
      if (o0->isRegIndex(0) && o1->isImm())
      {
        if (o0->getSize() == 2)
          _emitByte(0x66); // 16-bit.
        else if (o0->getSize() == 8)
          _emitByte(0x48); // REX.W.

        _emitByte((opReg << 3) | (0x04 + (o0->getSize() != 1)));
        _FINISHED_IMMEDIATE(o1, o0->getSize() <= 4 ? o0->getSize() : 4);
      }

      if (o0->isRegMem() && o1->isImm())
      {
        const Imm& imm = reinterpret_cast<const Imm&>(*o1);
        immSize = Util::isInt8(imm.getValue()) ? 1 : (o0->getSize() <= 4 ? o0->getSize() : 4);

        _emitX86RM(id->opCode[1] + (o0->getSize() != 1 ? (immSize != 1 ? 1 : 3) : 0),
          o0->getSize() == 2,
          o0->getSize() == 8,
          opReg, reinterpret_cast<const Operand&>(*o0),
          immSize, forceRexPrefix);
        _FINISHED_IMMEDIATE(&imm, immSize);
      }

      break;
    }

    case InstructionDescription::G_BSWAP:
    {
      if (o0->isReg())
      {
        const GPReg& dst = reinterpret_cast<const GPReg&>(*o0);

#if defined(ASMJIT_X64)
        _emitRexR(dst.getRegType() == REG_TYPE_GPQ, 1, dst.getRegCode(), forceRexPrefix);
#endif // ASMJIT_X64
        _emitByte(0x0F);
        _emitModR(1, dst.getRegCode());
        _FINISHED();
      }

      break;
    }

    case InstructionDescription::G_BT:
    {
      if (o0->isRegMem() && o1->isReg())
      {
        const Operand& dst = reinterpret_cast<const Operand&>(*o0);
        const GPReg& src = reinterpret_cast<const GPReg&>(*o1);

        _emitX86RM(id->opCode[0],
          src.isRegType(REG_TYPE_GPW),
          src.isRegType(REG_TYPE_GPQ),
          src.getRegCode(),
          dst,
          0, forceRexPrefix);
        _FINISHED();
      }

      if (o0->isRegMem() && o1->isImm())
      {
        const Operand& dst = reinterpret_cast<const Operand&>(*o0);
        const Imm& src = reinterpret_cast<const Imm&>(*o1);

        _emitX86RM(id->opCode[1],
          dst.getSize() == 2,
          dst.getSize() == 8,
          (uint8_t)id->opCodeR,
          dst,
          1, forceRexPrefix);
        _FINISHED_IMMEDIATE(o1, 1);
      }

      break;
    }

    case InstructionDescription::G_CALL:
    {
      if (o0->isRegTypeMem(REG_TYPE_GPN))
      {
        const Operand& dst = reinterpret_cast<const Operand&>(*o0);
        _emitX86RM(0xFF,
          0,
          0, 2, dst,
          0, forceRexPrefix);
        _FINISHED();
      }

      if (o0->isImm())
      {
        const Imm& imm = reinterpret_cast<const Imm&>(*o0);
        _emitByte(0xE8);
        _emitJmpOrCallReloc(InstructionDescription::G_CALL, (void*)imm.getValue());
        _FINISHED();
      }

      if (o0->isLabel())
      {
        LabelData& l_data = _labelData[reinterpret_cast<const Label*>(o0)->getId() & OPERAND_ID_VALUE_MASK];

        if (l_data.offset != -1)
        {
          // Bound label.
          static const sysint_t rel32_size = 5;
          sysint_t offs = l_data.offset - getOffset();

          ASMJIT_ASSERT(offs <= 0);

          _emitByte(0xE8);
          _emitInt32((int32_t)(offs - rel32_size));
        }
        else
        {
          // Non-bound label.
          _emitByte(0xE8);
          _emitDisplacement(l_data, -4, 4);
        }
        _FINISHED();
      }

      break;
    }

    case InstructionDescription::G_CRC32:
    {
      if (o0->isReg() && o1->isRegMem())
      {
        const GPReg& dst = reinterpret_cast<const GPReg&>(*o0);
        const Operand& src = reinterpret_cast<const Operand&>(*o1);
        ASMJIT_ASSERT(dst.getRegType() == REG_TYPE_GPD || dst.getRegType() == REG_TYPE_GPQ);

        _emitX86RM(id->opCode[0] + (src.getSize() != 1),
          src.getSize() == 2,
          dst.getRegType() == 8, dst.getRegCode(), src,
          0, forceRexPrefix);
        _FINISHED();
      }

      break;
    }

    case InstructionDescription::G_ENTER:
    {
      if (o0->isImm() && o1->isImm())
      {
        _emitByte(0xC8);
        _emitWord((uint16_t)(sysuint_t)reinterpret_cast<const Imm&>(*o2).getValue());
        _emitByte((uint8_t )(sysuint_t)reinterpret_cast<const Imm&>(*o1).getValue());
        _FINISHED();
      }
      break;
    }

    case InstructionDescription::G_IMUL:
    {
      // 1 operand
      if (o0->isRegMem() && o1->isNone() && o2->isNone())
      {
        const Operand& src = reinterpret_cast<const Operand&>(*o0);
        _emitX86RM(0xF6 + (src.getSize() != 1),
          src.getSize() == 2,
          src.getSize() == 8, 5, src,
          0, forceRexPrefix);
        _FINISHED();
      }
      // 2 operands
      else if (o0->isReg() && !o1->isNone() && o2->isNone())
      {
        const GPReg& dst = reinterpret_cast<const GPReg&>(*o0);
        ASMJIT_ASSERT(!dst.isRegType(REG_TYPE_GPW));

        if (o1->isRegMem())
        {
          const Operand& src = reinterpret_cast<const Operand&>(*o1);

          _emitX86RM(0x0FAF,
            dst.isRegType(REG_TYPE_GPW),
            dst.isRegType(REG_TYPE_GPQ), dst.getRegCode(), src,
            0, forceRexPrefix);
          _FINISHED();
        }
        else if (o1->isImm())
        {
          const Imm& imm = reinterpret_cast<const Imm&>(*o1);

          if (Util::isInt8(imm.getValue()))
          {
            _emitX86RM(0x6B,
              dst.isRegType(REG_TYPE_GPW),
              dst.isRegType(REG_TYPE_GPQ), dst.getRegCode(), dst,
              1, forceRexPrefix);
            _FINISHED_IMMEDIATE(&imm, 1);
          }
          else
          {
            immSize = dst.isRegType(REG_TYPE_GPW) ? 2 : 4;
            _emitX86RM(0x69,
              dst.isRegType(REG_TYPE_GPW),
              dst.isRegType(REG_TYPE_GPQ), dst.getRegCode(), dst,
              immSize, forceRexPrefix);
            _FINISHED_IMMEDIATE(&imm, immSize);
          }
        }
      }
      // 3 operands
      else if (o0->isReg() && o1->isRegMem() && o2->isImm())
      {
        const GPReg& dst = reinterpret_cast<const GPReg&>(*o0);
        const Operand& src = reinterpret_cast<const Operand&>(*o1);
        const Imm& imm = reinterpret_cast<const Imm&>(*o2);

        if (Util::isInt8(imm.getValue()))
        {
          _emitX86RM(0x6B,
            dst.isRegType(REG_TYPE_GPW),
            dst.isRegType(REG_TYPE_GPQ), dst.getRegCode(), src,
            1, forceRexPrefix);
          _FINISHED_IMMEDIATE(&imm, 1);
        }
        else
        {
          immSize = dst.isRegType(REG_TYPE_GPW) ? 2 : 4;
          _emitX86RM(0x69,
            dst.isRegType(REG_TYPE_GPW),
            dst.isRegType(REG_TYPE_GPQ), dst.getRegCode(), src,
            immSize, forceRexPrefix);
          _FINISHED_IMMEDIATE(&imm, immSize);
        }
      }

      break;
    }

    case InstructionDescription::G_INC_DEC:
    {
      if (o0->isRegMem())
      {
        const Operand& dst = reinterpret_cast<const Operand&>(*o0);

        // INC [r16|r32] in 64-bit mode is not encodable.
#if defined(ASMJIT_X86)
        if ((dst.isReg()) && (dst.isRegType(REG_TYPE_GPW) || dst.isRegType(REG_TYPE_GPD)))
        {
          _emitX86Inl(id->opCode[0],
            dst.isRegType(REG_TYPE_GPW),
            0, reinterpret_cast<const BaseReg&>(dst).getRegCode(),
            false);
          _FINISHED();
        }
#endif // ASMJIT_X86

        _emitX86RM(id->opCode[1] + (dst.getSize() != 1),
          dst.getSize() == 2,
          dst.getSize() == 8, (uint8_t)id->opCodeR, dst,
          0, forceRexPrefix);
        _FINISHED();
      }

      break;
    }

    case InstructionDescription::G_J:
    {
      if (o0->isLabel())
      {
        LabelData& l_data = _labelData[reinterpret_cast<const Label*>(o0)->getId() & OPERAND_ID_VALUE_MASK];

        uint32_t hint = (uint32_t)(o1->isImm() ? reinterpret_cast<const Imm&>(*o1).getValue() : 0);
        bool isShortJump = (_emitOptions & EMIT_OPTION_SHORT_JUMP) != 0;

        // Emit jump hint if configured for that.
        if ((hint & (HINT_TAKEN | HINT_NOT_TAKEN)) && (_properties & (1 << PROPERTY_JUMP_HINTS)))
        {
          if (hint & HINT_TAKEN)
            _emitByte(HINT_BYTE_VALUE_TAKEN);
          else if (hint & HINT_NOT_TAKEN)
            _emitByte(HINT_BYTE_VALUE_NOT_TAKEN);
        }

        if (l_data.offset != -1)
        {
          // Bound label.
          static const sysint_t rel8_size = 2;
          static const sysint_t rel32_size = 6;
          sysint_t offs = l_data.offset - getOffset();

          ASMJIT_ASSERT(offs <= 0);

          if (Util::isInt8(offs - rel8_size))
          {
            _emitByte(0x70 | (uint8_t)id->opCode[0]);
            _emitByte((uint8_t)(int8_t)(offs - rel8_size));

            // Change the emit options so logger can log instruction correctly.
            _emitOptions |= EMIT_OPTION_SHORT_JUMP;
          }
          else
          {
            if (isShortJump && _logger)
            {
              _logger->logString("*** ASSEMBLER WARNING: Emitting long conditional jump, but short jump instruction forced!\n");
              _emitOptions &= ~EMIT_OPTION_SHORT_JUMP;
            }

            _emitByte(0x0F);
            _emitByte(0x80 | (uint8_t)id->opCode[0]);
            _emitInt32((int32_t)(offs - rel32_size));
          }
        }
        else
        {
          // Non-bound label.
          if (isShortJump)
          {
            _emitByte(0x70 | (uint8_t)id->opCode[0]);
            _emitDisplacement(l_data, -1, 1);
          }
          else
          {
            _emitByte(0x0F);
            _emitByte(0x80 | (uint8_t)id->opCode[0]);
            _emitDisplacement(l_data, -4, 4);
          }
        }
        _FINISHED();
      }

      break;
    }

    case InstructionDescription::G_JMP:
    {
      if (o0->isRegMem())
      {
        const Operand& dst = reinterpret_cast<const Operand&>(*o0);

        _emitX86RM(0xFF,
          0,
          0, 4, dst,
          0, forceRexPrefix);
        _FINISHED();
      }

      if (o0->isImm())
      {
        const Imm& imm = reinterpret_cast<const Imm&>(*o0);
        _emitByte(0xE9);
        _emitJmpOrCallReloc(InstructionDescription::G_JMP, (void*)imm.getValue());
        _FINISHED();
      }

      if (o0->isLabel())
      {
        LabelData& l_data = _labelData[reinterpret_cast<const Label*>(o0)->getId() & OPERAND_ID_VALUE_MASK];
        bool isShortJump = (_emitOptions & EMIT_OPTION_SHORT_JUMP) != 0;

        if (l_data.offset != -1)
        {
          // Bound label.
          const sysint_t rel8_size = 2;
          const sysint_t rel32_size = 5;
          sysint_t offs = l_data.offset - getOffset();

          if (Util::isInt8(offs - rel8_size))
          {
            _emitByte(0xEB);
            _emitByte((uint8_t)(int8_t)(offs - rel8_size));

            // Change the emit options so logger can log instruction correctly.
            _emitOptions |= EMIT_OPTION_SHORT_JUMP;
          }
          else
          {
            if (isShortJump)
            {
              if (_logger)
              {
                _logger->logString("*** ASSEMBLER WARNING: Emitting long jump, but short jump instruction forced!\n");
                _emitOptions &= ~EMIT_OPTION_SHORT_JUMP;
              }
            }

            _emitByte(0xE9);
            _emitInt32((int32_t)(offs - rel32_size));
          }
        }
        else
        {
          // Non-bound label.
          if (isShortJump)
          {
            _emitByte(0xEB);
            _emitDisplacement(l_data, -1, 1);
          }
          else
          {
            _emitByte(0xE9);
            _emitDisplacement(l_data, -4, 4);
          }
        }
        _FINISHED();
      }

      break;
    }

    case InstructionDescription::G_LEA:
    {
      if (o0->isReg() && o1->isMem())
      {
        const GPReg& dst = reinterpret_cast<const GPReg&>(*o0);
        const Mem& src = reinterpret_cast<const Mem&>(*o1);

        // Size override prefix support.
        if (src.getSizePrefix())
        {
          _emitByte(0x67);
#if defined(ASMJIT_X86)
          memRegType = REG_TYPE_GPW;
#else
          memRegType = REG_TYPE_GPD;
#endif
        }

        _emitX86RM(0x8D,
          dst.isRegType(REG_TYPE_GPW),
          dst.isRegType(REG_TYPE_GPQ), dst.getRegCode(), src,
          0, forceRexPrefix);
        _FINISHED();
      }

      break;
    }

    case InstructionDescription::G_M:
    {
      if (o0->isMem())
      {
        _emitX86RM(id->opCode[0], 0, (uint8_t)id->opCode[1], (uint8_t)id->opCodeR, reinterpret_cast<const Mem&>(*o0), 0, forceRexPrefix);
        _FINISHED();
      }
      break;
    }

    case InstructionDescription::G_MOV:
    {
      const Operand& dst = *o0;
      const Operand& src = *o1;

      switch (dst.getType() << 4 | src.getType())
      {
        // Reg <- Reg/Mem
        case (OPERAND_REG << 4) | OPERAND_REG:
        {
          // Reg <- Sreg
          if (src.isRegType(REG_TYPE_SEGMENT))
          {
            ASMJIT_ASSERT(dst.isRegType(REG_TYPE_GPW) ||
                          dst.isRegType(REG_TYPE_GPD) ||
                          dst.isRegType(REG_TYPE_GPQ) );

            _emitX86RM(0x8C,
              dst.getSize() == 2,
              dst.getSize() == 8,
              reinterpret_cast<const SegmentReg&>(src).getRegCode(),
              reinterpret_cast<const Operand&>(dst),
              0, forceRexPrefix);
            _FINISHED();
          }

          // Sreg <- Reg/Mem
          if (dst.isRegType(REG_TYPE_SEGMENT))
          {
            ASMJIT_ASSERT(src.isRegType(REG_TYPE_GPW   ) ||
                          src.isRegType(REG_TYPE_GPD   ) ||
                          src.isRegType(REG_TYPE_GPQ   ) );

_Emit_Mov_Sreg_RM:
            _emitX86RM(0x8E,
              src.getSize() == 2,
              src.getSize() == 8,
              reinterpret_cast<const SegmentReg&>(dst).getRegCode(),
              reinterpret_cast<const Operand&>(src),
              0, forceRexPrefix);
            _FINISHED();
          }

          ASMJIT_ASSERT(src.isRegType(REG_TYPE_GPB_LO) ||
                        src.isRegType(REG_TYPE_GPB_HI) ||
                        src.isRegType(REG_TYPE_GPW   ) ||
                        src.isRegType(REG_TYPE_GPD   ) ||
                        src.isRegType(REG_TYPE_GPQ   ) );
          // ... fall through ...
        }
        case (OPERAND_REG << 4) | OPERAND_MEM:
        {
          // Sreg <- Mem
          if (dst.isRegType(REG_TYPE_SEGMENT))
          {
            goto _Emit_Mov_Sreg_RM;
          }

          ASMJIT_ASSERT(dst.isRegType(REG_TYPE_GPB_LO) ||
                        dst.isRegType(REG_TYPE_GPB_HI) ||
                        dst.isRegType(REG_TYPE_GPW   ) ||
                        dst.isRegType(REG_TYPE_GPD   ) ||
                        dst.isRegType(REG_TYPE_GPQ   ) );

          _emitX86RM(0x0000008A + (dst.getSize() != 1),
            dst.isRegType(REG_TYPE_GPW),
            dst.isRegType(REG_TYPE_GPQ),
            reinterpret_cast<const GPReg&>(dst).getRegCode(),
            reinterpret_cast<const Operand&>(src),
            0, forceRexPrefix);
          _FINISHED();
        }

        // Reg <- Imm
        case (OPERAND_REG << 4) | OPERAND_IMM:
        {
          const GPReg& dst = reinterpret_cast<const GPReg&>(*o0);
          const Imm& src = reinterpret_cast<const Imm&>(*o1);

          // In 64-bit mode the immediate can be 64-bits long if the
          // destination operand type is register (otherwise 32-bits).
          immSize = dst.getSize();

#if defined(ASMJIT_X64)
          // Optimize instruction size by using 32-bit immediate if value can
          // fit into it.
          if (immSize == 8 && Util::isInt32(src.getValue()))
          {
            _emitX86RM(0xC7,
              0, // 16BIT
              1, // REX.W
              0, // O
              dst,
              0, forceRexPrefix);
            immSize = 4;
          }
          else
          {
#endif // ASMJIT_X64
            _emitX86Inl((dst.getSize() == 1 ? 0xB0 : 0xB8),
              dst.isRegType(REG_TYPE_GPW),
              dst.isRegType(REG_TYPE_GPQ),
              dst.getRegCode(), forceRexPrefix);
#if defined(ASMJIT_X64)
          }
#endif // ASMJIT_X64

          _FINISHED_IMMEDIATE(&src, immSize);
        }

        // Mem <- Reg/Sreg
        case (OPERAND_MEM << 4) | OPERAND_REG:
        {
          if (src.isRegType(REG_TYPE_SEGMENT))
          {
            // Mem <- Sreg
            _emitX86RM(0x8C,
              dst.getSize() == 2,
              dst.getSize() == 8,
              reinterpret_cast<const SegmentReg&>(src).getRegCode(),
              reinterpret_cast<const Operand&>(dst),
              0, forceRexPrefix);
            _FINISHED();
          }
          else
          {
            // Mem <- Reg
            ASMJIT_ASSERT(src.isRegType(REG_TYPE_GPB_LO) ||
                          src.isRegType(REG_TYPE_GPB_HI) ||
                          src.isRegType(REG_TYPE_GPW   ) ||
                          src.isRegType(REG_TYPE_GPD   ) ||
                          src.isRegType(REG_TYPE_GPQ   ) );

            _emitX86RM(0x88 + (src.getSize() != 1),
              src.isRegType(REG_TYPE_GPW),
              src.isRegType(REG_TYPE_GPQ),
              reinterpret_cast<const GPReg&>(src).getRegCode(),
              reinterpret_cast<const Operand&>(dst),
              0, forceRexPrefix);
            _FINISHED();
          }
        }

        // Mem <- Imm
        case (OPERAND_MEM << 4) | OPERAND_IMM:
        {
          immSize = dst.getSize() <= 4 ? dst.getSize() : 4;

          _emitX86RM(0xC6 + (dst.getSize() != 1),
            dst.getSize() == 2,
            dst.getSize() == 8,
            0,
            reinterpret_cast<const Operand&>(dst),
            immSize, forceRexPrefix);
          _FINISHED_IMMEDIATE(&src, immSize);
        }
      }

      break;
    }

    case InstructionDescription::G_MOV_PTR:
    {
      if ((o0->isReg() && o1->isImm()) || (o0->isImm() && o1->isReg()))
      {
        bool reverse = o1->getType() == OPERAND_REG;
        uint8_t opCode = !reverse ? 0xA0 : 0xA2;
        const GPReg& reg = reinterpret_cast<const GPReg&>(!reverse ? *o0 : *o1);
        const Imm& imm = reinterpret_cast<const Imm&>(!reverse ? *o1 : *o0);

        if (reg.getRegIndex() != 0) goto illegalInstruction;

        if (reg.isRegType(REG_TYPE_GPW)) _emitByte(0x66);
#if defined(ASMJIT_X64)
        _emitRexR(reg.getSize() == 8, 0, 0, forceRexPrefix);
#endif // ASMJIT_X64
        _emitByte(opCode + (reg.getSize() != 1));
        _FINISHED_IMMEDIATE(&imm, sizeof(sysint_t));
      }

      break;
    }

    case InstructionDescription::G_MOVSX_MOVZX:
    {
      if (o0->isReg() && o1->isRegMem())
      {
        const GPReg& dst = reinterpret_cast<const GPReg&>(*o0);
        const Operand& src = reinterpret_cast<const Operand&>(*o1);

        if (dst.getSize() == 1) goto illegalInstruction;
        if (src.getSize() != 1 && src.getSize() != 2) goto illegalInstruction;
        if (src.getSize() == 2 && dst.getSize() == 2) goto illegalInstruction;

        _emitX86RM(id->opCode[0] + (src.getSize() != 1),
          dst.isRegType(REG_TYPE_GPW),
          dst.isRegType(REG_TYPE_GPQ),
          dst.getRegCode(),
          src,
          0, forceRexPrefix);
        _FINISHED();
      }

      break;
    }

#if defined(ASMJIT_X64)
    case InstructionDescription::G_MOVSXD:
    {
      if (o0->isReg() && o1->isRegMem())
      {
        const GPReg& dst = reinterpret_cast<const GPReg&>(*o0);
        const Operand& src = reinterpret_cast<const Operand&>(*o1);
        _emitX86RM(0x00000063,
          0,
          1, dst.getRegCode(), src,
          0, forceRexPrefix);
        _FINISHED();
      }

      break;
    }
#endif // ASMJIT_X64

    case InstructionDescription::G_PUSH:
    {
      // This section is only for immediates, memory/register operands are handled in G_POP.
      if (o0->isImm())
      {
        const Imm& imm = reinterpret_cast<const Imm&>(*o0);

        if (Util::isInt8(imm.getValue()))
        {
          _emitByte(0x6A);
          _FINISHED_IMMEDIATE(&imm, 1);
        }
        else
        {
          _emitByte(0x68);
          _FINISHED_IMMEDIATE(&imm, 4);
        }
      }

      // ... goto G_POP ...
    }

    case InstructionDescription::G_POP:
    {
      if (o0->isReg())
      {
        ASMJIT_ASSERT(o0->isRegType(REG_TYPE_GPW) || o0->isRegType(REG_TYPE_GPN));
        _emitX86Inl(id->opCode[0], o0->isRegType(REG_TYPE_GPW), 0, reinterpret_cast<const GPReg&>(*o0).getRegCode(), forceRexPrefix);
        _FINISHED();
      }

      if (o0->isMem())
      {
        _emitX86RM(id->opCode[1], o0->getSize() == 2, 0, (uint8_t)id->opCodeR, reinterpret_cast<const Operand&>(*o0), 0, forceRexPrefix);
        _FINISHED();
      }

      break;
    }

    case InstructionDescript…