PageRenderTime 222ms CodeModel.GetById 119ms app.highlight 84ms RepoModel.GetById 0ms app.codeStats 3ms

/3rd_party/llvm/lib/CodeGen/SelectionDAG/DAGCombiner.cpp

https://code.google.com/p/softart/
C++ | 11051 lines | 7632 code | 1358 blank | 2061 comment | 3191 complexity | 9168f24fe989eb6a6817a65e823725b7 MD5 | raw file
Possible License(s): LGPL-2.1, BSD-3-Clause, JSON, MPL-2.0-no-copyleft-exception, GPL-2.0, GPL-3.0, LGPL-3.0, BSD-2-Clause

Large files files are truncated, but you can click here to view the full file

   1//===-- DAGCombiner.cpp - Implement a DAG node combiner -------------------===//
   2//
   3//                     The LLVM Compiler Infrastructure
   4//
   5// This file is distributed under the University of Illinois Open Source
   6// License. See LICENSE.TXT for details.
   7//
   8//===----------------------------------------------------------------------===//
   9//
  10// This pass combines dag nodes to form fewer, simpler DAG nodes.  It can be run
  11// both before and after the DAG is legalized.
  12//
  13// This pass is not a substitute for the LLVM IR instcombine pass. This pass is
  14// primarily intended to handle simplification opportunities that are implicit
  15// in the LLVM IR and exposed by the various codegen lowering phases.
  16//
  17//===----------------------------------------------------------------------===//
  18
  19#define DEBUG_TYPE "dagcombine"
  20#include "llvm/CodeGen/SelectionDAG.h"
  21#include "llvm/ADT/SmallPtrSet.h"
  22#include "llvm/ADT/Statistic.h"
  23#include "llvm/Analysis/AliasAnalysis.h"
  24#include "llvm/CodeGen/MachineFrameInfo.h"
  25#include "llvm/CodeGen/MachineFunction.h"
  26#include "llvm/IR/DataLayout.h"
  27#include "llvm/IR/DerivedTypes.h"
  28#include "llvm/IR/Function.h"
  29#include "llvm/IR/LLVMContext.h"
  30#include "llvm/Support/CommandLine.h"
  31#include "llvm/Support/Debug.h"
  32#include "llvm/Support/ErrorHandling.h"
  33#include "llvm/Support/MathExtras.h"
  34#include "llvm/Support/raw_ostream.h"
  35#include "llvm/Target/TargetLowering.h"
  36#include "llvm/Target/TargetMachine.h"
  37#include "llvm/Target/TargetOptions.h"
  38#include "llvm/Target/TargetRegisterInfo.h"
  39#include "llvm/Target/TargetSubtargetInfo.h"
  40#include <algorithm>
  41using namespace llvm;
  42
  43STATISTIC(NodesCombined   , "Number of dag nodes combined");
  44STATISTIC(PreIndexedNodes , "Number of pre-indexed nodes created");
  45STATISTIC(PostIndexedNodes, "Number of post-indexed nodes created");
  46STATISTIC(OpsNarrowed     , "Number of load/op/store narrowed");
  47STATISTIC(LdStFP2Int      , "Number of fp load/store pairs transformed to int");
  48STATISTIC(SlicedLoads, "Number of load sliced");
  49
  50namespace {
  51  static cl::opt<bool>
  52    CombinerAA("combiner-alias-analysis", cl::Hidden,
  53               cl::desc("Turn on alias analysis during testing"));
  54
  55  static cl::opt<bool>
  56    CombinerGlobalAA("combiner-global-alias-analysis", cl::Hidden,
  57               cl::desc("Include global information in alias analysis"));
  58
  59  /// Hidden option to stress test load slicing, i.e., when this option
  60  /// is enabled, load slicing bypasses most of its profitability guards.
  61  static cl::opt<bool>
  62  StressLoadSlicing("combiner-stress-load-slicing", cl::Hidden,
  63                    cl::desc("Bypass the profitability model of load "
  64                             "slicing"),
  65                    cl::init(false));
  66
  67//------------------------------ DAGCombiner ---------------------------------//
  68
  69  class DAGCombiner {
  70    SelectionDAG &DAG;
  71    const TargetLowering &TLI;
  72    CombineLevel Level;
  73    CodeGenOpt::Level OptLevel;
  74    bool LegalOperations;
  75    bool LegalTypes;
  76    bool ForCodeSize;
  77
  78    // Worklist of all of the nodes that need to be simplified.
  79    //
  80    // This has the semantics that when adding to the worklist,
  81    // the item added must be next to be processed. It should
  82    // also only appear once. The naive approach to this takes
  83    // linear time.
  84    //
  85    // To reduce the insert/remove time to logarithmic, we use
  86    // a set and a vector to maintain our worklist.
  87    //
  88    // The set contains the items on the worklist, but does not
  89    // maintain the order they should be visited.
  90    //
  91    // The vector maintains the order nodes should be visited, but may
  92    // contain duplicate or removed nodes. When choosing a node to
  93    // visit, we pop off the order stack until we find an item that is
  94    // also in the contents set. All operations are O(log N).
  95    SmallPtrSet<SDNode*, 64> WorkListContents;
  96    SmallVector<SDNode*, 64> WorkListOrder;
  97
  98    // AA - Used for DAG load/store alias analysis.
  99    AliasAnalysis &AA;
 100
 101    /// AddUsersToWorkList - When an instruction is simplified, add all users of
 102    /// the instruction to the work lists because they might get more simplified
 103    /// now.
 104    ///
 105    void AddUsersToWorkList(SDNode *N) {
 106      for (SDNode::use_iterator UI = N->use_begin(), UE = N->use_end();
 107           UI != UE; ++UI)
 108        AddToWorkList(*UI);
 109    }
 110
 111    /// visit - call the node-specific routine that knows how to fold each
 112    /// particular type of node.
 113    SDValue visit(SDNode *N);
 114
 115  public:
 116    /// AddToWorkList - Add to the work list making sure its instance is at the
 117    /// back (next to be processed.)
 118    void AddToWorkList(SDNode *N) {
 119      WorkListContents.insert(N);
 120      WorkListOrder.push_back(N);
 121    }
 122
 123    /// removeFromWorkList - remove all instances of N from the worklist.
 124    ///
 125    void removeFromWorkList(SDNode *N) {
 126      WorkListContents.erase(N);
 127    }
 128
 129    SDValue CombineTo(SDNode *N, const SDValue *To, unsigned NumTo,
 130                      bool AddTo = true);
 131
 132    SDValue CombineTo(SDNode *N, SDValue Res, bool AddTo = true) {
 133      return CombineTo(N, &Res, 1, AddTo);
 134    }
 135
 136    SDValue CombineTo(SDNode *N, SDValue Res0, SDValue Res1,
 137                      bool AddTo = true) {
 138      SDValue To[] = { Res0, Res1 };
 139      return CombineTo(N, To, 2, AddTo);
 140    }
 141
 142    void CommitTargetLoweringOpt(const TargetLowering::TargetLoweringOpt &TLO);
 143
 144  private:
 145
 146    /// SimplifyDemandedBits - Check the specified integer node value to see if
 147    /// it can be simplified or if things it uses can be simplified by bit
 148    /// propagation.  If so, return true.
 149    bool SimplifyDemandedBits(SDValue Op) {
 150      unsigned BitWidth = Op.getValueType().getScalarType().getSizeInBits();
 151      APInt Demanded = APInt::getAllOnesValue(BitWidth);
 152      return SimplifyDemandedBits(Op, Demanded);
 153    }
 154
 155    bool SimplifyDemandedBits(SDValue Op, const APInt &Demanded);
 156
 157    bool CombineToPreIndexedLoadStore(SDNode *N);
 158    bool CombineToPostIndexedLoadStore(SDNode *N);
 159    bool SliceUpLoad(SDNode *N);
 160
 161    void ReplaceLoadWithPromotedLoad(SDNode *Load, SDNode *ExtLoad);
 162    SDValue PromoteOperand(SDValue Op, EVT PVT, bool &Replace);
 163    SDValue SExtPromoteOperand(SDValue Op, EVT PVT);
 164    SDValue ZExtPromoteOperand(SDValue Op, EVT PVT);
 165    SDValue PromoteIntBinOp(SDValue Op);
 166    SDValue PromoteIntShiftOp(SDValue Op);
 167    SDValue PromoteExtend(SDValue Op);
 168    bool PromoteLoad(SDValue Op);
 169
 170    void ExtendSetCCUses(const SmallVectorImpl<SDNode *> &SetCCs,
 171                         SDValue Trunc, SDValue ExtLoad, SDLoc DL,
 172                         ISD::NodeType ExtType);
 173
 174    /// combine - call the node-specific routine that knows how to fold each
 175    /// particular type of node. If that doesn't do anything, try the
 176    /// target-specific DAG combines.
 177    SDValue combine(SDNode *N);
 178
 179    // Visitation implementation - Implement dag node combining for different
 180    // node types.  The semantics are as follows:
 181    // Return Value:
 182    //   SDValue.getNode() == 0 - No change was made
 183    //   SDValue.getNode() == N - N was replaced, is dead and has been handled.
 184    //   otherwise              - N should be replaced by the returned Operand.
 185    //
 186    SDValue visitTokenFactor(SDNode *N);
 187    SDValue visitMERGE_VALUES(SDNode *N);
 188    SDValue visitADD(SDNode *N);
 189    SDValue visitSUB(SDNode *N);
 190    SDValue visitADDC(SDNode *N);
 191    SDValue visitSUBC(SDNode *N);
 192    SDValue visitADDE(SDNode *N);
 193    SDValue visitSUBE(SDNode *N);
 194    SDValue visitMUL(SDNode *N);
 195    SDValue visitSDIV(SDNode *N);
 196    SDValue visitUDIV(SDNode *N);
 197    SDValue visitSREM(SDNode *N);
 198    SDValue visitUREM(SDNode *N);
 199    SDValue visitMULHU(SDNode *N);
 200    SDValue visitMULHS(SDNode *N);
 201    SDValue visitSMUL_LOHI(SDNode *N);
 202    SDValue visitUMUL_LOHI(SDNode *N);
 203    SDValue visitSMULO(SDNode *N);
 204    SDValue visitUMULO(SDNode *N);
 205    SDValue visitSDIVREM(SDNode *N);
 206    SDValue visitUDIVREM(SDNode *N);
 207    SDValue visitAND(SDNode *N);
 208    SDValue visitOR(SDNode *N);
 209    SDValue visitXOR(SDNode *N);
 210    SDValue SimplifyVBinOp(SDNode *N);
 211    SDValue SimplifyVUnaryOp(SDNode *N);
 212    SDValue visitSHL(SDNode *N);
 213    SDValue visitSRA(SDNode *N);
 214    SDValue visitSRL(SDNode *N);
 215    SDValue visitCTLZ(SDNode *N);
 216    SDValue visitCTLZ_ZERO_UNDEF(SDNode *N);
 217    SDValue visitCTTZ(SDNode *N);
 218    SDValue visitCTTZ_ZERO_UNDEF(SDNode *N);
 219    SDValue visitCTPOP(SDNode *N);
 220    SDValue visitSELECT(SDNode *N);
 221    SDValue visitVSELECT(SDNode *N);
 222    SDValue visitSELECT_CC(SDNode *N);
 223    SDValue visitSETCC(SDNode *N);
 224    SDValue visitSIGN_EXTEND(SDNode *N);
 225    SDValue visitZERO_EXTEND(SDNode *N);
 226    SDValue visitANY_EXTEND(SDNode *N);
 227    SDValue visitSIGN_EXTEND_INREG(SDNode *N);
 228    SDValue visitTRUNCATE(SDNode *N);
 229    SDValue visitBITCAST(SDNode *N);
 230    SDValue visitBUILD_PAIR(SDNode *N);
 231    SDValue visitFADD(SDNode *N);
 232    SDValue visitFSUB(SDNode *N);
 233    SDValue visitFMUL(SDNode *N);
 234    SDValue visitFMA(SDNode *N);
 235    SDValue visitFDIV(SDNode *N);
 236    SDValue visitFREM(SDNode *N);
 237    SDValue visitFCOPYSIGN(SDNode *N);
 238    SDValue visitSINT_TO_FP(SDNode *N);
 239    SDValue visitUINT_TO_FP(SDNode *N);
 240    SDValue visitFP_TO_SINT(SDNode *N);
 241    SDValue visitFP_TO_UINT(SDNode *N);
 242    SDValue visitFP_ROUND(SDNode *N);
 243    SDValue visitFP_ROUND_INREG(SDNode *N);
 244    SDValue visitFP_EXTEND(SDNode *N);
 245    SDValue visitFNEG(SDNode *N);
 246    SDValue visitFABS(SDNode *N);
 247    SDValue visitFCEIL(SDNode *N);
 248    SDValue visitFTRUNC(SDNode *N);
 249    SDValue visitFFLOOR(SDNode *N);
 250    SDValue visitBRCOND(SDNode *N);
 251    SDValue visitBR_CC(SDNode *N);
 252    SDValue visitLOAD(SDNode *N);
 253    SDValue visitSTORE(SDNode *N);
 254    SDValue visitINSERT_VECTOR_ELT(SDNode *N);
 255    SDValue visitEXTRACT_VECTOR_ELT(SDNode *N);
 256    SDValue visitBUILD_VECTOR(SDNode *N);
 257    SDValue visitCONCAT_VECTORS(SDNode *N);
 258    SDValue visitEXTRACT_SUBVECTOR(SDNode *N);
 259    SDValue visitVECTOR_SHUFFLE(SDNode *N);
 260
 261    SDValue XformToShuffleWithZero(SDNode *N);
 262    SDValue ReassociateOps(unsigned Opc, SDLoc DL, SDValue LHS, SDValue RHS);
 263
 264    SDValue visitShiftByConstant(SDNode *N, unsigned Amt);
 265
 266    bool SimplifySelectOps(SDNode *SELECT, SDValue LHS, SDValue RHS);
 267    SDValue SimplifyBinOpWithSameOpcodeHands(SDNode *N);
 268    SDValue SimplifySelect(SDLoc DL, SDValue N0, SDValue N1, SDValue N2);
 269    SDValue SimplifySelectCC(SDLoc DL, SDValue N0, SDValue N1, SDValue N2,
 270                             SDValue N3, ISD::CondCode CC,
 271                             bool NotExtCompare = false);
 272    SDValue SimplifySetCC(EVT VT, SDValue N0, SDValue N1, ISD::CondCode Cond,
 273                          SDLoc DL, bool foldBooleans = true);
 274    SDValue SimplifyNodeWithTwoResults(SDNode *N, unsigned LoOp,
 275                                         unsigned HiOp);
 276    SDValue CombineConsecutiveLoads(SDNode *N, EVT VT);
 277    SDValue ConstantFoldBITCASTofBUILD_VECTOR(SDNode *, EVT);
 278    SDValue BuildSDIV(SDNode *N);
 279    SDValue BuildUDIV(SDNode *N);
 280    SDValue MatchBSwapHWordLow(SDNode *N, SDValue N0, SDValue N1,
 281                               bool DemandHighBits = true);
 282    SDValue MatchBSwapHWord(SDNode *N, SDValue N0, SDValue N1);
 283    SDNode *MatchRotate(SDValue LHS, SDValue RHS, SDLoc DL);
 284    SDValue ReduceLoadWidth(SDNode *N);
 285    SDValue ReduceLoadOpStoreWidth(SDNode *N);
 286    SDValue TransformFPLoadStorePair(SDNode *N);
 287    SDValue reduceBuildVecExtToExtBuildVec(SDNode *N);
 288    SDValue reduceBuildVecConvertToConvertBuildVec(SDNode *N);
 289
 290    SDValue GetDemandedBits(SDValue V, const APInt &Mask);
 291
 292    /// GatherAllAliases - Walk up chain skipping non-aliasing memory nodes,
 293    /// looking for aliasing nodes and adding them to the Aliases vector.
 294    void GatherAllAliases(SDNode *N, SDValue OriginalChain,
 295                          SmallVectorImpl<SDValue> &Aliases);
 296
 297    /// isAlias - Return true if there is any possibility that the two addresses
 298    /// overlap.
 299    bool isAlias(SDValue Ptr1, int64_t Size1, bool IsVolatile1,
 300                 const Value *SrcValue1, int SrcValueOffset1,
 301                 unsigned SrcValueAlign1,
 302                 const MDNode *TBAAInfo1,
 303                 SDValue Ptr2, int64_t Size2, bool IsVolatile2,
 304                 const Value *SrcValue2, int SrcValueOffset2,
 305                 unsigned SrcValueAlign2,
 306                 const MDNode *TBAAInfo2) const;
 307
 308    /// isAlias - Return true if there is any possibility that the two addresses
 309    /// overlap.
 310    bool isAlias(LSBaseSDNode *Op0, LSBaseSDNode *Op1);
 311
 312    /// FindAliasInfo - Extracts the relevant alias information from the memory
 313    /// node.  Returns true if the operand was a load.
 314    bool FindAliasInfo(SDNode *N,
 315                       SDValue &Ptr, int64_t &Size, bool &IsVolatile,
 316                       const Value *&SrcValue, int &SrcValueOffset,
 317                       unsigned &SrcValueAlignment,
 318                       const MDNode *&TBAAInfo) const;
 319
 320    /// FindBetterChain - Walk up chain skipping non-aliasing memory nodes,
 321    /// looking for a better chain (aliasing node.)
 322    SDValue FindBetterChain(SDNode *N, SDValue Chain);
 323
 324    /// Merge consecutive store operations into a wide store.
 325    /// This optimization uses wide integers or vectors when possible.
 326    /// \return True if some memory operations were changed.
 327    bool MergeConsecutiveStores(StoreSDNode *N);
 328
 329  public:
 330    DAGCombiner(SelectionDAG &D, AliasAnalysis &A, CodeGenOpt::Level OL)
 331        : DAG(D), TLI(D.getTargetLoweringInfo()), Level(BeforeLegalizeTypes),
 332          OptLevel(OL), LegalOperations(false), LegalTypes(false), AA(A) {
 333      AttributeSet FnAttrs =
 334          DAG.getMachineFunction().getFunction()->getAttributes();
 335      ForCodeSize =
 336          FnAttrs.hasAttribute(AttributeSet::FunctionIndex,
 337                               Attribute::OptimizeForSize) ||
 338          FnAttrs.hasAttribute(AttributeSet::FunctionIndex, Attribute::MinSize);
 339    }
 340
 341    /// Run - runs the dag combiner on all nodes in the work list
 342    void Run(CombineLevel AtLevel);
 343
 344    SelectionDAG &getDAG() const { return DAG; }
 345
 346    /// getShiftAmountTy - Returns a type large enough to hold any valid
 347    /// shift amount - before type legalization these can be huge.
 348    EVT getShiftAmountTy(EVT LHSTy) {
 349      assert(LHSTy.isInteger() && "Shift amount is not an integer type!");
 350      if (LHSTy.isVector())
 351        return LHSTy;
 352      return LegalTypes ? TLI.getScalarShiftAmountTy(LHSTy)
 353                        : TLI.getPointerTy();
 354    }
 355
 356    /// isTypeLegal - This method returns true if we are running before type
 357    /// legalization or if the specified VT is legal.
 358    bool isTypeLegal(const EVT &VT) {
 359      if (!LegalTypes) return true;
 360      return TLI.isTypeLegal(VT);
 361    }
 362
 363    /// getSetCCResultType - Convenience wrapper around
 364    /// TargetLowering::getSetCCResultType
 365    EVT getSetCCResultType(EVT VT) const {
 366      return TLI.getSetCCResultType(*DAG.getContext(), VT);
 367    }
 368  };
 369}
 370
 371
 372namespace {
 373/// WorkListRemover - This class is a DAGUpdateListener that removes any deleted
 374/// nodes from the worklist.
 375class WorkListRemover : public SelectionDAG::DAGUpdateListener {
 376  DAGCombiner &DC;
 377public:
 378  explicit WorkListRemover(DAGCombiner &dc)
 379    : SelectionDAG::DAGUpdateListener(dc.getDAG()), DC(dc) {}
 380
 381  virtual void NodeDeleted(SDNode *N, SDNode *E) {
 382    DC.removeFromWorkList(N);
 383  }
 384};
 385}
 386
 387//===----------------------------------------------------------------------===//
 388//  TargetLowering::DAGCombinerInfo implementation
 389//===----------------------------------------------------------------------===//
 390
 391void TargetLowering::DAGCombinerInfo::AddToWorklist(SDNode *N) {
 392  ((DAGCombiner*)DC)->AddToWorkList(N);
 393}
 394
 395void TargetLowering::DAGCombinerInfo::RemoveFromWorklist(SDNode *N) {
 396  ((DAGCombiner*)DC)->removeFromWorkList(N);
 397}
 398
 399SDValue TargetLowering::DAGCombinerInfo::
 400CombineTo(SDNode *N, const std::vector<SDValue> &To, bool AddTo) {
 401  return ((DAGCombiner*)DC)->CombineTo(N, &To[0], To.size(), AddTo);
 402}
 403
 404SDValue TargetLowering::DAGCombinerInfo::
 405CombineTo(SDNode *N, SDValue Res, bool AddTo) {
 406  return ((DAGCombiner*)DC)->CombineTo(N, Res, AddTo);
 407}
 408
 409
 410SDValue TargetLowering::DAGCombinerInfo::
 411CombineTo(SDNode *N, SDValue Res0, SDValue Res1, bool AddTo) {
 412  return ((DAGCombiner*)DC)->CombineTo(N, Res0, Res1, AddTo);
 413}
 414
 415void TargetLowering::DAGCombinerInfo::
 416CommitTargetLoweringOpt(const TargetLowering::TargetLoweringOpt &TLO) {
 417  return ((DAGCombiner*)DC)->CommitTargetLoweringOpt(TLO);
 418}
 419
 420//===----------------------------------------------------------------------===//
 421// Helper Functions
 422//===----------------------------------------------------------------------===//
 423
 424/// isNegatibleForFree - Return 1 if we can compute the negated form of the
 425/// specified expression for the same cost as the expression itself, or 2 if we
 426/// can compute the negated form more cheaply than the expression itself.
 427static char isNegatibleForFree(SDValue Op, bool LegalOperations,
 428                               const TargetLowering &TLI,
 429                               const TargetOptions *Options,
 430                               unsigned Depth = 0) {
 431  // fneg is removable even if it has multiple uses.
 432  if (Op.getOpcode() == ISD::FNEG) return 2;
 433
 434  // Don't allow anything with multiple uses.
 435  if (!Op.hasOneUse()) return 0;
 436
 437  // Don't recurse exponentially.
 438  if (Depth > 6) return 0;
 439
 440  switch (Op.getOpcode()) {
 441  default: return false;
 442  case ISD::ConstantFP:
 443    // Don't invert constant FP values after legalize.  The negated constant
 444    // isn't necessarily legal.
 445    return LegalOperations ? 0 : 1;
 446  case ISD::FADD:
 447    // FIXME: determine better conditions for this xform.
 448    if (!Options->UnsafeFPMath) return 0;
 449
 450    // After operation legalization, it might not be legal to create new FSUBs.
 451    if (LegalOperations &&
 452        !TLI.isOperationLegalOrCustom(ISD::FSUB,  Op.getValueType()))
 453      return 0;
 454
 455    // fold (fneg (fadd A, B)) -> (fsub (fneg A), B)
 456    if (char V = isNegatibleForFree(Op.getOperand(0), LegalOperations, TLI,
 457                                    Options, Depth + 1))
 458      return V;
 459    // fold (fneg (fadd A, B)) -> (fsub (fneg B), A)
 460    return isNegatibleForFree(Op.getOperand(1), LegalOperations, TLI, Options,
 461                              Depth + 1);
 462  case ISD::FSUB:
 463    // We can't turn -(A-B) into B-A when we honor signed zeros.
 464    if (!Options->UnsafeFPMath) return 0;
 465
 466    // fold (fneg (fsub A, B)) -> (fsub B, A)
 467    return 1;
 468
 469  case ISD::FMUL:
 470  case ISD::FDIV:
 471    if (Options->HonorSignDependentRoundingFPMath()) return 0;
 472
 473    // fold (fneg (fmul X, Y)) -> (fmul (fneg X), Y) or (fmul X, (fneg Y))
 474    if (char V = isNegatibleForFree(Op.getOperand(0), LegalOperations, TLI,
 475                                    Options, Depth + 1))
 476      return V;
 477
 478    return isNegatibleForFree(Op.getOperand(1), LegalOperations, TLI, Options,
 479                              Depth + 1);
 480
 481  case ISD::FP_EXTEND:
 482  case ISD::FP_ROUND:
 483  case ISD::FSIN:
 484    return isNegatibleForFree(Op.getOperand(0), LegalOperations, TLI, Options,
 485                              Depth + 1);
 486  }
 487}
 488
 489/// GetNegatedExpression - If isNegatibleForFree returns true, this function
 490/// returns the newly negated expression.
 491static SDValue GetNegatedExpression(SDValue Op, SelectionDAG &DAG,
 492                                    bool LegalOperations, unsigned Depth = 0) {
 493  // fneg is removable even if it has multiple uses.
 494  if (Op.getOpcode() == ISD::FNEG) return Op.getOperand(0);
 495
 496  // Don't allow anything with multiple uses.
 497  assert(Op.hasOneUse() && "Unknown reuse!");
 498
 499  assert(Depth <= 6 && "GetNegatedExpression doesn't match isNegatibleForFree");
 500  switch (Op.getOpcode()) {
 501  default: llvm_unreachable("Unknown code");
 502  case ISD::ConstantFP: {
 503    APFloat V = cast<ConstantFPSDNode>(Op)->getValueAPF();
 504    V.changeSign();
 505    return DAG.getConstantFP(V, Op.getValueType());
 506  }
 507  case ISD::FADD:
 508    // FIXME: determine better conditions for this xform.
 509    assert(DAG.getTarget().Options.UnsafeFPMath);
 510
 511    // fold (fneg (fadd A, B)) -> (fsub (fneg A), B)
 512    if (isNegatibleForFree(Op.getOperand(0), LegalOperations,
 513                           DAG.getTargetLoweringInfo(),
 514                           &DAG.getTarget().Options, Depth+1))
 515      return DAG.getNode(ISD::FSUB, SDLoc(Op), Op.getValueType(),
 516                         GetNegatedExpression(Op.getOperand(0), DAG,
 517                                              LegalOperations, Depth+1),
 518                         Op.getOperand(1));
 519    // fold (fneg (fadd A, B)) -> (fsub (fneg B), A)
 520    return DAG.getNode(ISD::FSUB, SDLoc(Op), Op.getValueType(),
 521                       GetNegatedExpression(Op.getOperand(1), DAG,
 522                                            LegalOperations, Depth+1),
 523                       Op.getOperand(0));
 524  case ISD::FSUB:
 525    // We can't turn -(A-B) into B-A when we honor signed zeros.
 526    assert(DAG.getTarget().Options.UnsafeFPMath);
 527
 528    // fold (fneg (fsub 0, B)) -> B
 529    if (ConstantFPSDNode *N0CFP = dyn_cast<ConstantFPSDNode>(Op.getOperand(0)))
 530      if (N0CFP->getValueAPF().isZero())
 531        return Op.getOperand(1);
 532
 533    // fold (fneg (fsub A, B)) -> (fsub B, A)
 534    return DAG.getNode(ISD::FSUB, SDLoc(Op), Op.getValueType(),
 535                       Op.getOperand(1), Op.getOperand(0));
 536
 537  case ISD::FMUL:
 538  case ISD::FDIV:
 539    assert(!DAG.getTarget().Options.HonorSignDependentRoundingFPMath());
 540
 541    // fold (fneg (fmul X, Y)) -> (fmul (fneg X), Y)
 542    if (isNegatibleForFree(Op.getOperand(0), LegalOperations,
 543                           DAG.getTargetLoweringInfo(),
 544                           &DAG.getTarget().Options, Depth+1))
 545      return DAG.getNode(Op.getOpcode(), SDLoc(Op), Op.getValueType(),
 546                         GetNegatedExpression(Op.getOperand(0), DAG,
 547                                              LegalOperations, Depth+1),
 548                         Op.getOperand(1));
 549
 550    // fold (fneg (fmul X, Y)) -> (fmul X, (fneg Y))
 551    return DAG.getNode(Op.getOpcode(), SDLoc(Op), Op.getValueType(),
 552                       Op.getOperand(0),
 553                       GetNegatedExpression(Op.getOperand(1), DAG,
 554                                            LegalOperations, Depth+1));
 555
 556  case ISD::FP_EXTEND:
 557  case ISD::FSIN:
 558    return DAG.getNode(Op.getOpcode(), SDLoc(Op), Op.getValueType(),
 559                       GetNegatedExpression(Op.getOperand(0), DAG,
 560                                            LegalOperations, Depth+1));
 561  case ISD::FP_ROUND:
 562      return DAG.getNode(ISD::FP_ROUND, SDLoc(Op), Op.getValueType(),
 563                         GetNegatedExpression(Op.getOperand(0), DAG,
 564                                              LegalOperations, Depth+1),
 565                         Op.getOperand(1));
 566  }
 567}
 568
 569
 570// isSetCCEquivalent - Return true if this node is a setcc, or is a select_cc
 571// that selects between the values 1 and 0, making it equivalent to a setcc.
 572// Also, set the incoming LHS, RHS, and CC references to the appropriate
 573// nodes based on the type of node we are checking.  This simplifies life a
 574// bit for the callers.
 575static bool isSetCCEquivalent(SDValue N, SDValue &LHS, SDValue &RHS,
 576                              SDValue &CC) {
 577  if (N.getOpcode() == ISD::SETCC) {
 578    LHS = N.getOperand(0);
 579    RHS = N.getOperand(1);
 580    CC  = N.getOperand(2);
 581    return true;
 582  }
 583  if (N.getOpcode() == ISD::SELECT_CC &&
 584      N.getOperand(2).getOpcode() == ISD::Constant &&
 585      N.getOperand(3).getOpcode() == ISD::Constant &&
 586      cast<ConstantSDNode>(N.getOperand(2))->getAPIntValue() == 1 &&
 587      cast<ConstantSDNode>(N.getOperand(3))->isNullValue()) {
 588    LHS = N.getOperand(0);
 589    RHS = N.getOperand(1);
 590    CC  = N.getOperand(4);
 591    return true;
 592  }
 593  return false;
 594}
 595
 596// isOneUseSetCC - Return true if this is a SetCC-equivalent operation with only
 597// one use.  If this is true, it allows the users to invert the operation for
 598// free when it is profitable to do so.
 599static bool isOneUseSetCC(SDValue N) {
 600  SDValue N0, N1, N2;
 601  if (isSetCCEquivalent(N, N0, N1, N2) && N.getNode()->hasOneUse())
 602    return true;
 603  return false;
 604}
 605
 606SDValue DAGCombiner::ReassociateOps(unsigned Opc, SDLoc DL,
 607                                    SDValue N0, SDValue N1) {
 608  EVT VT = N0.getValueType();
 609  if (N0.getOpcode() == Opc && isa<ConstantSDNode>(N0.getOperand(1))) {
 610    if (isa<ConstantSDNode>(N1)) {
 611      // reassoc. (op (op x, c1), c2) -> (op x, (op c1, c2))
 612      SDValue OpNode =
 613        DAG.FoldConstantArithmetic(Opc, VT,
 614                                   cast<ConstantSDNode>(N0.getOperand(1)),
 615                                   cast<ConstantSDNode>(N1));
 616      return DAG.getNode(Opc, DL, VT, N0.getOperand(0), OpNode);
 617    }
 618    if (N0.hasOneUse()) {
 619      // reassoc. (op (op x, c1), y) -> (op (op x, y), c1) iff x+c1 has one use
 620      SDValue OpNode = DAG.getNode(Opc, SDLoc(N0), VT,
 621                                   N0.getOperand(0), N1);
 622      AddToWorkList(OpNode.getNode());
 623      return DAG.getNode(Opc, DL, VT, OpNode, N0.getOperand(1));
 624    }
 625  }
 626
 627  if (N1.getOpcode() == Opc && isa<ConstantSDNode>(N1.getOperand(1))) {
 628    if (isa<ConstantSDNode>(N0)) {
 629      // reassoc. (op c2, (op x, c1)) -> (op x, (op c1, c2))
 630      SDValue OpNode =
 631        DAG.FoldConstantArithmetic(Opc, VT,
 632                                   cast<ConstantSDNode>(N1.getOperand(1)),
 633                                   cast<ConstantSDNode>(N0));
 634      return DAG.getNode(Opc, DL, VT, N1.getOperand(0), OpNode);
 635    }
 636    if (N1.hasOneUse()) {
 637      // reassoc. (op y, (op x, c1)) -> (op (op x, y), c1) iff x+c1 has one use
 638      SDValue OpNode = DAG.getNode(Opc, SDLoc(N0), VT,
 639                                   N1.getOperand(0), N0);
 640      AddToWorkList(OpNode.getNode());
 641      return DAG.getNode(Opc, DL, VT, OpNode, N1.getOperand(1));
 642    }
 643  }
 644
 645  return SDValue();
 646}
 647
 648SDValue DAGCombiner::CombineTo(SDNode *N, const SDValue *To, unsigned NumTo,
 649                               bool AddTo) {
 650  assert(N->getNumValues() == NumTo && "Broken CombineTo call!");
 651  ++NodesCombined;
 652  DEBUG(dbgs() << "\nReplacing.1 ";
 653        N->dump(&DAG);
 654        dbgs() << "\nWith: ";
 655        To[0].getNode()->dump(&DAG);
 656        dbgs() << " and " << NumTo-1 << " other values\n";
 657        for (unsigned i = 0, e = NumTo; i != e; ++i)
 658          assert((!To[i].getNode() ||
 659                  N->getValueType(i) == To[i].getValueType()) &&
 660                 "Cannot combine value to value of different type!"));
 661  WorkListRemover DeadNodes(*this);
 662  DAG.ReplaceAllUsesWith(N, To);
 663  if (AddTo) {
 664    // Push the new nodes and any users onto the worklist
 665    for (unsigned i = 0, e = NumTo; i != e; ++i) {
 666      if (To[i].getNode()) {
 667        AddToWorkList(To[i].getNode());
 668        AddUsersToWorkList(To[i].getNode());
 669      }
 670    }
 671  }
 672
 673  // Finally, if the node is now dead, remove it from the graph.  The node
 674  // may not be dead if the replacement process recursively simplified to
 675  // something else needing this node.
 676  if (N->use_empty()) {
 677    // Nodes can be reintroduced into the worklist.  Make sure we do not
 678    // process a node that has been replaced.
 679    removeFromWorkList(N);
 680
 681    // Finally, since the node is now dead, remove it from the graph.
 682    DAG.DeleteNode(N);
 683  }
 684  return SDValue(N, 0);
 685}
 686
 687void DAGCombiner::
 688CommitTargetLoweringOpt(const TargetLowering::TargetLoweringOpt &TLO) {
 689  // Replace all uses.  If any nodes become isomorphic to other nodes and
 690  // are deleted, make sure to remove them from our worklist.
 691  WorkListRemover DeadNodes(*this);
 692  DAG.ReplaceAllUsesOfValueWith(TLO.Old, TLO.New);
 693
 694  // Push the new node and any (possibly new) users onto the worklist.
 695  AddToWorkList(TLO.New.getNode());
 696  AddUsersToWorkList(TLO.New.getNode());
 697
 698  // Finally, if the node is now dead, remove it from the graph.  The node
 699  // may not be dead if the replacement process recursively simplified to
 700  // something else needing this node.
 701  if (TLO.Old.getNode()->use_empty()) {
 702    removeFromWorkList(TLO.Old.getNode());
 703
 704    // If the operands of this node are only used by the node, they will now
 705    // be dead.  Make sure to visit them first to delete dead nodes early.
 706    for (unsigned i = 0, e = TLO.Old.getNode()->getNumOperands(); i != e; ++i)
 707      if (TLO.Old.getNode()->getOperand(i).getNode()->hasOneUse())
 708        AddToWorkList(TLO.Old.getNode()->getOperand(i).getNode());
 709
 710    DAG.DeleteNode(TLO.Old.getNode());
 711  }
 712}
 713
 714/// SimplifyDemandedBits - Check the specified integer node value to see if
 715/// it can be simplified or if things it uses can be simplified by bit
 716/// propagation.  If so, return true.
 717bool DAGCombiner::SimplifyDemandedBits(SDValue Op, const APInt &Demanded) {
 718  TargetLowering::TargetLoweringOpt TLO(DAG, LegalTypes, LegalOperations);
 719  APInt KnownZero, KnownOne;
 720  if (!TLI.SimplifyDemandedBits(Op, Demanded, KnownZero, KnownOne, TLO))
 721    return false;
 722
 723  // Revisit the node.
 724  AddToWorkList(Op.getNode());
 725
 726  // Replace the old value with the new one.
 727  ++NodesCombined;
 728  DEBUG(dbgs() << "\nReplacing.2 ";
 729        TLO.Old.getNode()->dump(&DAG);
 730        dbgs() << "\nWith: ";
 731        TLO.New.getNode()->dump(&DAG);
 732        dbgs() << '\n');
 733
 734  CommitTargetLoweringOpt(TLO);
 735  return true;
 736}
 737
 738void DAGCombiner::ReplaceLoadWithPromotedLoad(SDNode *Load, SDNode *ExtLoad) {
 739  SDLoc dl(Load);
 740  EVT VT = Load->getValueType(0);
 741  SDValue Trunc = DAG.getNode(ISD::TRUNCATE, dl, VT, SDValue(ExtLoad, 0));
 742
 743  DEBUG(dbgs() << "\nReplacing.9 ";
 744        Load->dump(&DAG);
 745        dbgs() << "\nWith: ";
 746        Trunc.getNode()->dump(&DAG);
 747        dbgs() << '\n');
 748  WorkListRemover DeadNodes(*this);
 749  DAG.ReplaceAllUsesOfValueWith(SDValue(Load, 0), Trunc);
 750  DAG.ReplaceAllUsesOfValueWith(SDValue(Load, 1), SDValue(ExtLoad, 1));
 751  removeFromWorkList(Load);
 752  DAG.DeleteNode(Load);
 753  AddToWorkList(Trunc.getNode());
 754}
 755
 756SDValue DAGCombiner::PromoteOperand(SDValue Op, EVT PVT, bool &Replace) {
 757  Replace = false;
 758  SDLoc dl(Op);
 759  if (LoadSDNode *LD = dyn_cast<LoadSDNode>(Op)) {
 760    EVT MemVT = LD->getMemoryVT();
 761    ISD::LoadExtType ExtType = ISD::isNON_EXTLoad(LD)
 762      ? (TLI.isLoadExtLegal(ISD::ZEXTLOAD, MemVT) ? ISD::ZEXTLOAD
 763                                                  : ISD::EXTLOAD)
 764      : LD->getExtensionType();
 765    Replace = true;
 766    return DAG.getExtLoad(ExtType, dl, PVT,
 767                          LD->getChain(), LD->getBasePtr(),
 768                          MemVT, LD->getMemOperand());
 769  }
 770
 771  unsigned Opc = Op.getOpcode();
 772  switch (Opc) {
 773  default: break;
 774  case ISD::AssertSext:
 775    return DAG.getNode(ISD::AssertSext, dl, PVT,
 776                       SExtPromoteOperand(Op.getOperand(0), PVT),
 777                       Op.getOperand(1));
 778  case ISD::AssertZext:
 779    return DAG.getNode(ISD::AssertZext, dl, PVT,
 780                       ZExtPromoteOperand(Op.getOperand(0), PVT),
 781                       Op.getOperand(1));
 782  case ISD::Constant: {
 783    unsigned ExtOpc =
 784      Op.getValueType().isByteSized() ? ISD::SIGN_EXTEND : ISD::ZERO_EXTEND;
 785    return DAG.getNode(ExtOpc, dl, PVT, Op);
 786  }
 787  }
 788
 789  if (!TLI.isOperationLegal(ISD::ANY_EXTEND, PVT))
 790    return SDValue();
 791  return DAG.getNode(ISD::ANY_EXTEND, dl, PVT, Op);
 792}
 793
 794SDValue DAGCombiner::SExtPromoteOperand(SDValue Op, EVT PVT) {
 795  if (!TLI.isOperationLegal(ISD::SIGN_EXTEND_INREG, PVT))
 796    return SDValue();
 797  EVT OldVT = Op.getValueType();
 798  SDLoc dl(Op);
 799  bool Replace = false;
 800  SDValue NewOp = PromoteOperand(Op, PVT, Replace);
 801  if (NewOp.getNode() == 0)
 802    return SDValue();
 803  AddToWorkList(NewOp.getNode());
 804
 805  if (Replace)
 806    ReplaceLoadWithPromotedLoad(Op.getNode(), NewOp.getNode());
 807  return DAG.getNode(ISD::SIGN_EXTEND_INREG, dl, NewOp.getValueType(), NewOp,
 808                     DAG.getValueType(OldVT));
 809}
 810
 811SDValue DAGCombiner::ZExtPromoteOperand(SDValue Op, EVT PVT) {
 812  EVT OldVT = Op.getValueType();
 813  SDLoc dl(Op);
 814  bool Replace = false;
 815  SDValue NewOp = PromoteOperand(Op, PVT, Replace);
 816  if (NewOp.getNode() == 0)
 817    return SDValue();
 818  AddToWorkList(NewOp.getNode());
 819
 820  if (Replace)
 821    ReplaceLoadWithPromotedLoad(Op.getNode(), NewOp.getNode());
 822  return DAG.getZeroExtendInReg(NewOp, dl, OldVT);
 823}
 824
 825/// PromoteIntBinOp - Promote the specified integer binary operation if the
 826/// target indicates it is beneficial. e.g. On x86, it's usually better to
 827/// promote i16 operations to i32 since i16 instructions are longer.
 828SDValue DAGCombiner::PromoteIntBinOp(SDValue Op) {
 829  if (!LegalOperations)
 830    return SDValue();
 831
 832  EVT VT = Op.getValueType();
 833  if (VT.isVector() || !VT.isInteger())
 834    return SDValue();
 835
 836  // If operation type is 'undesirable', e.g. i16 on x86, consider
 837  // promoting it.
 838  unsigned Opc = Op.getOpcode();
 839  if (TLI.isTypeDesirableForOp(Opc, VT))
 840    return SDValue();
 841
 842  EVT PVT = VT;
 843  // Consult target whether it is a good idea to promote this operation and
 844  // what's the right type to promote it to.
 845  if (TLI.IsDesirableToPromoteOp(Op, PVT)) {
 846    assert(PVT != VT && "Don't know what type to promote to!");
 847
 848    bool Replace0 = false;
 849    SDValue N0 = Op.getOperand(0);
 850    SDValue NN0 = PromoteOperand(N0, PVT, Replace0);
 851    if (NN0.getNode() == 0)
 852      return SDValue();
 853
 854    bool Replace1 = false;
 855    SDValue N1 = Op.getOperand(1);
 856    SDValue NN1;
 857    if (N0 == N1)
 858      NN1 = NN0;
 859    else {
 860      NN1 = PromoteOperand(N1, PVT, Replace1);
 861      if (NN1.getNode() == 0)
 862        return SDValue();
 863    }
 864
 865    AddToWorkList(NN0.getNode());
 866    if (NN1.getNode())
 867      AddToWorkList(NN1.getNode());
 868
 869    if (Replace0)
 870      ReplaceLoadWithPromotedLoad(N0.getNode(), NN0.getNode());
 871    if (Replace1)
 872      ReplaceLoadWithPromotedLoad(N1.getNode(), NN1.getNode());
 873
 874    DEBUG(dbgs() << "\nPromoting ";
 875          Op.getNode()->dump(&DAG));
 876    SDLoc dl(Op);
 877    return DAG.getNode(ISD::TRUNCATE, dl, VT,
 878                       DAG.getNode(Opc, dl, PVT, NN0, NN1));
 879  }
 880  return SDValue();
 881}
 882
 883/// PromoteIntShiftOp - Promote the specified integer shift operation if the
 884/// target indicates it is beneficial. e.g. On x86, it's usually better to
 885/// promote i16 operations to i32 since i16 instructions are longer.
 886SDValue DAGCombiner::PromoteIntShiftOp(SDValue Op) {
 887  if (!LegalOperations)
 888    return SDValue();
 889
 890  EVT VT = Op.getValueType();
 891  if (VT.isVector() || !VT.isInteger())
 892    return SDValue();
 893
 894  // If operation type is 'undesirable', e.g. i16 on x86, consider
 895  // promoting it.
 896  unsigned Opc = Op.getOpcode();
 897  if (TLI.isTypeDesirableForOp(Opc, VT))
 898    return SDValue();
 899
 900  EVT PVT = VT;
 901  // Consult target whether it is a good idea to promote this operation and
 902  // what's the right type to promote it to.
 903  if (TLI.IsDesirableToPromoteOp(Op, PVT)) {
 904    assert(PVT != VT && "Don't know what type to promote to!");
 905
 906    bool Replace = false;
 907    SDValue N0 = Op.getOperand(0);
 908    if (Opc == ISD::SRA)
 909      N0 = SExtPromoteOperand(Op.getOperand(0), PVT);
 910    else if (Opc == ISD::SRL)
 911      N0 = ZExtPromoteOperand(Op.getOperand(0), PVT);
 912    else
 913      N0 = PromoteOperand(N0, PVT, Replace);
 914    if (N0.getNode() == 0)
 915      return SDValue();
 916
 917    AddToWorkList(N0.getNode());
 918    if (Replace)
 919      ReplaceLoadWithPromotedLoad(Op.getOperand(0).getNode(), N0.getNode());
 920
 921    DEBUG(dbgs() << "\nPromoting ";
 922          Op.getNode()->dump(&DAG));
 923    SDLoc dl(Op);
 924    return DAG.getNode(ISD::TRUNCATE, dl, VT,
 925                       DAG.getNode(Opc, dl, PVT, N0, Op.getOperand(1)));
 926  }
 927  return SDValue();
 928}
 929
 930SDValue DAGCombiner::PromoteExtend(SDValue Op) {
 931  if (!LegalOperations)
 932    return SDValue();
 933
 934  EVT VT = Op.getValueType();
 935  if (VT.isVector() || !VT.isInteger())
 936    return SDValue();
 937
 938  // If operation type is 'undesirable', e.g. i16 on x86, consider
 939  // promoting it.
 940  unsigned Opc = Op.getOpcode();
 941  if (TLI.isTypeDesirableForOp(Opc, VT))
 942    return SDValue();
 943
 944  EVT PVT = VT;
 945  // Consult target whether it is a good idea to promote this operation and
 946  // what's the right type to promote it to.
 947  if (TLI.IsDesirableToPromoteOp(Op, PVT)) {
 948    assert(PVT != VT && "Don't know what type to promote to!");
 949    // fold (aext (aext x)) -> (aext x)
 950    // fold (aext (zext x)) -> (zext x)
 951    // fold (aext (sext x)) -> (sext x)
 952    DEBUG(dbgs() << "\nPromoting ";
 953          Op.getNode()->dump(&DAG));
 954    return DAG.getNode(Op.getOpcode(), SDLoc(Op), VT, Op.getOperand(0));
 955  }
 956  return SDValue();
 957}
 958
 959bool DAGCombiner::PromoteLoad(SDValue Op) {
 960  if (!LegalOperations)
 961    return false;
 962
 963  EVT VT = Op.getValueType();
 964  if (VT.isVector() || !VT.isInteger())
 965    return false;
 966
 967  // If operation type is 'undesirable', e.g. i16 on x86, consider
 968  // promoting it.
 969  unsigned Opc = Op.getOpcode();
 970  if (TLI.isTypeDesirableForOp(Opc, VT))
 971    return false;
 972
 973  EVT PVT = VT;
 974  // Consult target whether it is a good idea to promote this operation and
 975  // what's the right type to promote it to.
 976  if (TLI.IsDesirableToPromoteOp(Op, PVT)) {
 977    assert(PVT != VT && "Don't know what type to promote to!");
 978
 979    SDLoc dl(Op);
 980    SDNode *N = Op.getNode();
 981    LoadSDNode *LD = cast<LoadSDNode>(N);
 982    EVT MemVT = LD->getMemoryVT();
 983    ISD::LoadExtType ExtType = ISD::isNON_EXTLoad(LD)
 984      ? (TLI.isLoadExtLegal(ISD::ZEXTLOAD, MemVT) ? ISD::ZEXTLOAD
 985                                                  : ISD::EXTLOAD)
 986      : LD->getExtensionType();
 987    SDValue NewLD = DAG.getExtLoad(ExtType, dl, PVT,
 988                                   LD->getChain(), LD->getBasePtr(),
 989                                   MemVT, LD->getMemOperand());
 990    SDValue Result = DAG.getNode(ISD::TRUNCATE, dl, VT, NewLD);
 991
 992    DEBUG(dbgs() << "\nPromoting ";
 993          N->dump(&DAG);
 994          dbgs() << "\nTo: ";
 995          Result.getNode()->dump(&DAG);
 996          dbgs() << '\n');
 997    WorkListRemover DeadNodes(*this);
 998    DAG.ReplaceAllUsesOfValueWith(SDValue(N, 0), Result);
 999    DAG.ReplaceAllUsesOfValueWith(SDValue(N, 1), NewLD.getValue(1));
1000    removeFromWorkList(N);
1001    DAG.DeleteNode(N);
1002    AddToWorkList(Result.getNode());
1003    return true;
1004  }
1005  return false;
1006}
1007
1008
1009//===----------------------------------------------------------------------===//
1010//  Main DAG Combiner implementation
1011//===----------------------------------------------------------------------===//
1012
1013void DAGCombiner::Run(CombineLevel AtLevel) {
1014  // set the instance variables, so that the various visit routines may use it.
1015  Level = AtLevel;
1016  LegalOperations = Level >= AfterLegalizeVectorOps;
1017  LegalTypes = Level >= AfterLegalizeTypes;
1018
1019  // Add all the dag nodes to the worklist.
1020  for (SelectionDAG::allnodes_iterator I = DAG.allnodes_begin(),
1021       E = DAG.allnodes_end(); I != E; ++I)
1022    AddToWorkList(I);
1023
1024  // Create a dummy node (which is not added to allnodes), that adds a reference
1025  // to the root node, preventing it from being deleted, and tracking any
1026  // changes of the root.
1027  HandleSDNode Dummy(DAG.getRoot());
1028
1029  // The root of the dag may dangle to deleted nodes until the dag combiner is
1030  // done.  Set it to null to avoid confusion.
1031  DAG.setRoot(SDValue());
1032
1033  // while the worklist isn't empty, find a node and
1034  // try and combine it.
1035  while (!WorkListContents.empty()) {
1036    SDNode *N;
1037    // The WorkListOrder holds the SDNodes in order, but it may contain
1038    // duplicates.
1039    // In order to avoid a linear scan, we use a set (O(log N)) to hold what the
1040    // worklist *should* contain, and check the node we want to visit is should
1041    // actually be visited.
1042    do {
1043      N = WorkListOrder.pop_back_val();
1044    } while (!WorkListContents.erase(N));
1045
1046    // If N has no uses, it is dead.  Make sure to revisit all N's operands once
1047    // N is deleted from the DAG, since they too may now be dead or may have a
1048    // reduced number of uses, allowing other xforms.
1049    if (N->use_empty() && N != &Dummy) {
1050      for (unsigned i = 0, e = N->getNumOperands(); i != e; ++i)
1051        AddToWorkList(N->getOperand(i).getNode());
1052
1053      DAG.DeleteNode(N);
1054      continue;
1055    }
1056
1057    SDValue RV = combine(N);
1058
1059    if (RV.getNode() == 0)
1060      continue;
1061
1062    ++NodesCombined;
1063
1064    // If we get back the same node we passed in, rather than a new node or
1065    // zero, we know that the node must have defined multiple values and
1066    // CombineTo was used.  Since CombineTo takes care of the worklist
1067    // mechanics for us, we have no work to do in this case.
1068    if (RV.getNode() == N)
1069      continue;
1070
1071    assert(N->getOpcode() != ISD::DELETED_NODE &&
1072           RV.getNode()->getOpcode() != ISD::DELETED_NODE &&
1073           "Node was deleted but visit returned new node!");
1074
1075    DEBUG(dbgs() << "\nReplacing.3 ";
1076          N->dump(&DAG);
1077          dbgs() << "\nWith: ";
1078          RV.getNode()->dump(&DAG);
1079          dbgs() << '\n');
1080
1081    // Transfer debug value.
1082    DAG.TransferDbgValues(SDValue(N, 0), RV);
1083    WorkListRemover DeadNodes(*this);
1084    if (N->getNumValues() == RV.getNode()->getNumValues())
1085      DAG.ReplaceAllUsesWith(N, RV.getNode());
1086    else {
1087      assert(N->getValueType(0) == RV.getValueType() &&
1088             N->getNumValues() == 1 && "Type mismatch");
1089      SDValue OpV = RV;
1090      DAG.ReplaceAllUsesWith(N, &OpV);
1091    }
1092
1093    // Push the new node and any users onto the worklist
1094    AddToWorkList(RV.getNode());
1095    AddUsersToWorkList(RV.getNode());
1096
1097    // Add any uses of the old node to the worklist in case this node is the
1098    // last one that uses them.  They may become dead after this node is
1099    // deleted.
1100    for (unsigned i = 0, e = N->getNumOperands(); i != e; ++i)
1101      AddToWorkList(N->getOperand(i).getNode());
1102
1103    // Finally, if the node is now dead, remove it from the graph.  The node
1104    // may not be dead if the replacement process recursively simplified to
1105    // something else needing this node.
1106    if (N->use_empty()) {
1107      // Nodes can be reintroduced into the worklist.  Make sure we do not
1108      // process a node that has been replaced.
1109      removeFromWorkList(N);
1110
1111      // Finally, since the node is now dead, remove it from the graph.
1112      DAG.DeleteNode(N);
1113    }
1114  }
1115
1116  // If the root changed (e.g. it was a dead load, update the root).
1117  DAG.setRoot(Dummy.getValue());
1118  DAG.RemoveDeadNodes();
1119}
1120
1121SDValue DAGCombiner::visit(SDNode *N) {
1122  switch (N->getOpcode()) {
1123  default: break;
1124  case ISD::TokenFactor:        return visitTokenFactor(N);
1125  case ISD::MERGE_VALUES:       return visitMERGE_VALUES(N);
1126  case ISD::ADD:                return visitADD(N);
1127  case ISD::SUB:                return visitSUB(N);
1128  case ISD::ADDC:               return visitADDC(N);
1129  case ISD::SUBC:               return visitSUBC(N);
1130  case ISD::ADDE:               return visitADDE(N);
1131  case ISD::SUBE:               return visitSUBE(N);
1132  case ISD::MUL:                return visitMUL(N);
1133  case ISD::SDIV:               return visitSDIV(N);
1134  case ISD::UDIV:               return visitUDIV(N);
1135  case ISD::SREM:               return visitSREM(N);
1136  case ISD::UREM:               return visitUREM(N);
1137  case ISD::MULHU:              return visitMULHU(N);
1138  case ISD::MULHS:              return visitMULHS(N);
1139  case ISD::SMUL_LOHI:          return visitSMUL_LOHI(N);
1140  case ISD::UMUL_LOHI:          return visitUMUL_LOHI(N);
1141  case ISD::SMULO:              return visitSMULO(N);
1142  case ISD::UMULO:              return visitUMULO(N);
1143  case ISD::SDIVREM:            return visitSDIVREM(N);
1144  case ISD::UDIVREM:            return visitUDIVREM(N);
1145  case ISD::AND:                return visitAND(N);
1146  case ISD::OR:                 return visitOR(N);
1147  case ISD::XOR:                return visitXOR(N);
1148  case ISD::SHL:                return visitSHL(N);
1149  case ISD::SRA:                return visitSRA(N);
1150  case ISD::SRL:                return visitSRL(N);
1151  case ISD::CTLZ:               return visitCTLZ(N);
1152  case ISD::CTLZ_ZERO_UNDEF:    return visitCTLZ_ZERO_UNDEF(N);
1153  case ISD::CTTZ:               return visitCTTZ(N);
1154  case ISD::CTTZ_ZERO_UNDEF:    return visitCTTZ_ZERO_UNDEF(N);
1155  case ISD::CTPOP:              return visitCTPOP(N);
1156  case ISD::SELECT:             return visitSELECT(N);
1157  case ISD::VSELECT:            return visitVSELECT(N);
1158  case ISD::SELECT_CC:          return visitSELECT_CC(N);
1159  case ISD::SETCC:              return visitSETCC(N);
1160  case ISD::SIGN_EXTEND:        return visitSIGN_EXTEND(N);
1161  case ISD::ZERO_EXTEND:        return visitZERO_EXTEND(N);
1162  case ISD::ANY_EXTEND:         return visitANY_EXTEND(N);
1163  case ISD::SIGN_EXTEND_INREG:  return visitSIGN_EXTEND_INREG(N);
1164  case ISD::TRUNCATE:           return visitTRUNCATE(N);
1165  case ISD::BITCAST:            return visitBITCAST(N);
1166  case ISD::BUILD_PAIR:         return visitBUILD_PAIR(N);
1167  case ISD::FADD:               return visitFADD(N);
1168  case ISD::FSUB:               return visitFSUB(N);
1169  case ISD::FMUL:               return visitFMUL(N);
1170  case ISD::FMA:                return visitFMA(N);
1171  case ISD::FDIV:               return visitFDIV(N);
1172  case ISD::FREM:               return visitFREM(N);
1173  case ISD::FCOPYSIGN:          return visitFCOPYSIGN(N);
1174  case ISD::SINT_TO_FP:         return visitSINT_TO_FP(N);
1175  case ISD::UINT_TO_FP:         return visitUINT_TO_FP(N);
1176  case ISD::FP_TO_SINT:         return visitFP_TO_SINT(N);
1177  case ISD::FP_TO_UINT:         return visitFP_TO_UINT(N);
1178  case ISD::FP_ROUND:           return visitFP_ROUND(N);
1179  case ISD::FP_ROUND_INREG:     return visitFP_ROUND_INREG(N);
1180  case ISD::FP_EXTEND:          return visitFP_EXTEND(N);
1181  case ISD::FNEG:               return visitFNEG(N);
1182  case ISD::FABS:               return visitFABS(N);
1183  case ISD::FFLOOR:             return visitFFLOOR(N);
1184  case ISD::FCEIL:              return visitFCEIL(N);
1185  case ISD::FTRUNC:             return visitFTRUNC(N);
1186  case ISD::BRCOND:             return visitBRCOND(N);
1187  case ISD::BR_CC:              return visitBR_CC(N);
1188  case ISD::LOAD:               return visitLOAD(N);
1189  case ISD::STORE:              return visitSTORE(N);
1190  case ISD::INSERT_VECTOR_ELT:  return visitINSERT_VECTOR_ELT(N);
1191  case ISD::EXTRACT_VECTOR_ELT: return visitEXTRACT_VECTOR_ELT(N);
1192  case ISD::BUILD_VECTOR:       return visitBUILD_VECTOR(N);
1193  case ISD::CONCAT_VECTORS:     return visitCONCAT_VECTORS(N);
1194  case ISD::EXTRACT_SUBVECTOR:  return visitEXTRACT_SUBVECTOR(N);
1195  case ISD::VECTOR_SHUFFLE:     return visitVECTOR_SHUFFLE(N);
1196  }
1197  return SDValue();
1198}
1199
1200SDValue DAGCombiner::combine(SDNode *N) {
1201  SDValue RV = visit(N);
1202
1203  // If nothing happened, try a target-specific DAG combine.
1204  if (RV.getNode() == 0) {
1205    assert(N->getOpcode() != ISD::DELETED_NODE &&
1206           "Node was deleted but visit returned NULL!");
1207
1208    if (N->getOpcode() >= ISD::BUILTIN_OP_END ||
1209        TLI.hasTargetDAGCombine((ISD::NodeType)N->getOpcode())) {
1210
1211      // Expose the DAG combiner to the target combiner impls.
1212      TargetLowering::DAGCombinerInfo
1213        DagCombineInfo(DAG, Level, false, this);
1214
1215      RV = TLI.PerformDAGCombine(N, DagCombineInfo);
1216    }
1217  }
1218
1219  // If nothing happened still, try promoting the operation.
1220  if (RV.getNode() == 0) {
1221    switch (N->getOpcode()) {
1222    default: break;
1223    case ISD::ADD:
1224    case ISD::SUB:
1225    case ISD::MUL:
1226    case ISD::AND:
1227    case ISD::OR:
1228    case ISD::XOR:
1229      RV = PromoteIntBinOp(SDValue(N, 0));
1230      break;
1231    case ISD::SHL:
1232    case ISD::SRA:
1233    case ISD::SRL:
1234      RV = PromoteIntShiftOp(SDValue(N, 0));
1235      break;
1236    case ISD::SIGN_EXTEND:
1237    case ISD::ZERO_EXTEND:
1238    case ISD::ANY_EXTEND:
1239      RV = PromoteExtend(SDValue(N, 0));
1240      break;
1241    case ISD::LOAD:
1242      if (PromoteLoad(SDValue(N, 0)))
1243        RV = SDValue(N, 0);
1244      break;
1245    }
1246  }
1247
1248  // If N is a commutative binary node, try commuting it to enable more
1249  // sdisel CSE.
1250  if (RV.getNode() == 0 &&
1251      SelectionDAG::isCommutativeBinOp(N->getOpcode()) &&
1252      N->getNumValues() == 1) {
1253    SDValue N0 = N->getOperand(0);
1254    SDValue N1 = N->getOperand(1);
1255
1256    // Constant operands are canonicalized to RHS.
1257    if (isa<ConstantSDNode>(N0) || !isa<ConstantSDNode>(N1)) {
1258      SDValue Ops[] = { N1, N0 };
1259      SDNode *CSENode = DAG.getNodeIfExists(N->getOpcode(), N->getVTList(),
1260                                            Ops, 2);
1261      if (CSENode)
1262        return SDValue(CSENode, 0);
1263    }
1264  }
1265
1266  return RV;
1267}
1268
1269/// getInputChainForNode - Given a node, return its input chain if it has one,
1270/// otherwise return a null sd operand.
1271static SDValue getInputChainForNode(SDNode *N) {
1272  if (unsigned NumOps = N->getNumOperands()) {
1273    if (N->getOperand(0).getValueType() == MVT::Other)
1274      return N->getOperand(0);
1275    if (N->getOperand(NumOps-1).getValueType() == MVT::Other)
1276      return N->getOperand(NumOps-1);
1277    for (unsigned i = 1; i < NumOps-1; ++i)
1278      if (N->getOperand(i).getValueType() == MVT::Other)
1279        return N->getOperand(i);
1280  }
1281  return SDValue();
1282}
1283
1284SDValue DAGCombiner::visitTokenFactor(SDNode *N) {
1285  // If N has two operands, where one has an input chain equal to the other,
1286  // the 'other' chain is redundant.
1287  if (N->getNumOperands() == 2) {
1288    if (getInputChainForNode(N->getOperand(0).getNode()) == N->getOperand(1))
1289      return N->getOperand(0);
1290    if (getInputChainForNode(N->getOperand(1).getNode()) == N->getOperand(0))
1291      return N->getOperand(1);
1292  }
1293
1294  SmallVector<SDNode *, 8> TFs;     // List of token factors to visit.
1295  SmallVector<SDValue, 8> Ops;    // Ops for replacing token factor.
1296  SmallPtrSet<SDNode*, 16> SeenOps;
1297  bool Changed = false;             // If we should replace this token factor.
1298
1299  // Start out with this token factor.
1300  TFs.push_back(N);
1301
1302  // Iterate through token factors.  The TFs grows when new token factors are
1303  // encountered.
1304  for (unsigned i = 0; i < TFs.size(); ++i) {
1305    SDNode *TF = TFs[i];
1306
1307    // Check each of the operands.
1308    for (unsigned i = 0, ie = TF->getNumOperands(); i != ie; ++i) {
1309      SDValue Op = TF->getOperand(i);
1310
1311      switch (Op.getOpcode()) {
1312      case ISD::EntryToken:
1313        // Entry tokens don't need to be added to the list. They are
1314        // rededundant.
1315        Changed = true;
1316        break;
1317
1318      case ISD::TokenFactor:
1319        if (Op.hasOneUse() &&
1320            std::find(TFs.begin(), TFs.end(), Op.getNode()) == TFs.end()) {
1321          // Q…

Large files files are truncated, but you can click here to view the full file