/3rd_party/llvm/lib/CodeGen/SelectionDAG/DAGCombiner.cpp
https://code.google.com/p/softart/ · C++ · 11051 lines · 7632 code · 1358 blank · 2061 comment · 3191 complexity · 9168f24fe989eb6a6817a65e823725b7 MD5 · raw file
Large files are truncated click here to view the full file
- //===-- DAGCombiner.cpp - Implement a DAG node combiner -------------------===//
- //
- // The LLVM Compiler Infrastructure
- //
- // This file is distributed under the University of Illinois Open Source
- // License. See LICENSE.TXT for details.
- //
- //===----------------------------------------------------------------------===//
- //
- // This pass combines dag nodes to form fewer, simpler DAG nodes. It can be run
- // both before and after the DAG is legalized.
- //
- // This pass is not a substitute for the LLVM IR instcombine pass. This pass is
- // primarily intended to handle simplification opportunities that are implicit
- // in the LLVM IR and exposed by the various codegen lowering phases.
- //
- //===----------------------------------------------------------------------===//
- #define DEBUG_TYPE "dagcombine"
- #include "llvm/CodeGen/SelectionDAG.h"
- #include "llvm/ADT/SmallPtrSet.h"
- #include "llvm/ADT/Statistic.h"
- #include "llvm/Analysis/AliasAnalysis.h"
- #include "llvm/CodeGen/MachineFrameInfo.h"
- #include "llvm/CodeGen/MachineFunction.h"
- #include "llvm/IR/DataLayout.h"
- #include "llvm/IR/DerivedTypes.h"
- #include "llvm/IR/Function.h"
- #include "llvm/IR/LLVMContext.h"
- #include "llvm/Support/CommandLine.h"
- #include "llvm/Support/Debug.h"
- #include "llvm/Support/ErrorHandling.h"
- #include "llvm/Support/MathExtras.h"
- #include "llvm/Support/raw_ostream.h"
- #include "llvm/Target/TargetLowering.h"
- #include "llvm/Target/TargetMachine.h"
- #include "llvm/Target/TargetOptions.h"
- #include "llvm/Target/TargetRegisterInfo.h"
- #include "llvm/Target/TargetSubtargetInfo.h"
- #include <algorithm>
- using namespace llvm;
- STATISTIC(NodesCombined , "Number of dag nodes combined");
- STATISTIC(PreIndexedNodes , "Number of pre-indexed nodes created");
- STATISTIC(PostIndexedNodes, "Number of post-indexed nodes created");
- STATISTIC(OpsNarrowed , "Number of load/op/store narrowed");
- STATISTIC(LdStFP2Int , "Number of fp load/store pairs transformed to int");
- STATISTIC(SlicedLoads, "Number of load sliced");
- namespace {
- static cl::opt<bool>
- CombinerAA("combiner-alias-analysis", cl::Hidden,
- cl::desc("Turn on alias analysis during testing"));
- static cl::opt<bool>
- CombinerGlobalAA("combiner-global-alias-analysis", cl::Hidden,
- cl::desc("Include global information in alias analysis"));
- /// Hidden option to stress test load slicing, i.e., when this option
- /// is enabled, load slicing bypasses most of its profitability guards.
- static cl::opt<bool>
- StressLoadSlicing("combiner-stress-load-slicing", cl::Hidden,
- cl::desc("Bypass the profitability model of load "
- "slicing"),
- cl::init(false));
- //------------------------------ DAGCombiner ---------------------------------//
- class DAGCombiner {
- SelectionDAG &DAG;
- const TargetLowering &TLI;
- CombineLevel Level;
- CodeGenOpt::Level OptLevel;
- bool LegalOperations;
- bool LegalTypes;
- bool ForCodeSize;
- // Worklist of all of the nodes that need to be simplified.
- //
- // This has the semantics that when adding to the worklist,
- // the item added must be next to be processed. It should
- // also only appear once. The naive approach to this takes
- // linear time.
- //
- // To reduce the insert/remove time to logarithmic, we use
- // a set and a vector to maintain our worklist.
- //
- // The set contains the items on the worklist, but does not
- // maintain the order they should be visited.
- //
- // The vector maintains the order nodes should be visited, but may
- // contain duplicate or removed nodes. When choosing a node to
- // visit, we pop off the order stack until we find an item that is
- // also in the contents set. All operations are O(log N).
- SmallPtrSet<SDNode*, 64> WorkListContents;
- SmallVector<SDNode*, 64> WorkListOrder;
- // AA - Used for DAG load/store alias analysis.
- AliasAnalysis &AA;
- /// AddUsersToWorkList - When an instruction is simplified, add all users of
- /// the instruction to the work lists because they might get more simplified
- /// now.
- ///
- void AddUsersToWorkList(SDNode *N) {
- for (SDNode::use_iterator UI = N->use_begin(), UE = N->use_end();
- UI != UE; ++UI)
- AddToWorkList(*UI);
- }
- /// visit - call the node-specific routine that knows how to fold each
- /// particular type of node.
- SDValue visit(SDNode *N);
- public:
- /// AddToWorkList - Add to the work list making sure its instance is at the
- /// back (next to be processed.)
- void AddToWorkList(SDNode *N) {
- WorkListContents.insert(N);
- WorkListOrder.push_back(N);
- }
- /// removeFromWorkList - remove all instances of N from the worklist.
- ///
- void removeFromWorkList(SDNode *N) {
- WorkListContents.erase(N);
- }
- SDValue CombineTo(SDNode *N, const SDValue *To, unsigned NumTo,
- bool AddTo = true);
- SDValue CombineTo(SDNode *N, SDValue Res, bool AddTo = true) {
- return CombineTo(N, &Res, 1, AddTo);
- }
- SDValue CombineTo(SDNode *N, SDValue Res0, SDValue Res1,
- bool AddTo = true) {
- SDValue To[] = { Res0, Res1 };
- return CombineTo(N, To, 2, AddTo);
- }
- void CommitTargetLoweringOpt(const TargetLowering::TargetLoweringOpt &TLO);
- private:
- /// SimplifyDemandedBits - Check the specified integer node value to see if
- /// it can be simplified or if things it uses can be simplified by bit
- /// propagation. If so, return true.
- bool SimplifyDemandedBits(SDValue Op) {
- unsigned BitWidth = Op.getValueType().getScalarType().getSizeInBits();
- APInt Demanded = APInt::getAllOnesValue(BitWidth);
- return SimplifyDemandedBits(Op, Demanded);
- }
- bool SimplifyDemandedBits(SDValue Op, const APInt &Demanded);
- bool CombineToPreIndexedLoadStore(SDNode *N);
- bool CombineToPostIndexedLoadStore(SDNode *N);
- bool SliceUpLoad(SDNode *N);
- void ReplaceLoadWithPromotedLoad(SDNode *Load, SDNode *ExtLoad);
- SDValue PromoteOperand(SDValue Op, EVT PVT, bool &Replace);
- SDValue SExtPromoteOperand(SDValue Op, EVT PVT);
- SDValue ZExtPromoteOperand(SDValue Op, EVT PVT);
- SDValue PromoteIntBinOp(SDValue Op);
- SDValue PromoteIntShiftOp(SDValue Op);
- SDValue PromoteExtend(SDValue Op);
- bool PromoteLoad(SDValue Op);
- void ExtendSetCCUses(const SmallVectorImpl<SDNode *> &SetCCs,
- SDValue Trunc, SDValue ExtLoad, SDLoc DL,
- ISD::NodeType ExtType);
- /// combine - call the node-specific routine that knows how to fold each
- /// particular type of node. If that doesn't do anything, try the
- /// target-specific DAG combines.
- SDValue combine(SDNode *N);
- // Visitation implementation - Implement dag node combining for different
- // node types. The semantics are as follows:
- // Return Value:
- // SDValue.getNode() == 0 - No change was made
- // SDValue.getNode() == N - N was replaced, is dead and has been handled.
- // otherwise - N should be replaced by the returned Operand.
- //
- SDValue visitTokenFactor(SDNode *N);
- SDValue visitMERGE_VALUES(SDNode *N);
- SDValue visitADD(SDNode *N);
- SDValue visitSUB(SDNode *N);
- SDValue visitADDC(SDNode *N);
- SDValue visitSUBC(SDNode *N);
- SDValue visitADDE(SDNode *N);
- SDValue visitSUBE(SDNode *N);
- SDValue visitMUL(SDNode *N);
- SDValue visitSDIV(SDNode *N);
- SDValue visitUDIV(SDNode *N);
- SDValue visitSREM(SDNode *N);
- SDValue visitUREM(SDNode *N);
- SDValue visitMULHU(SDNode *N);
- SDValue visitMULHS(SDNode *N);
- SDValue visitSMUL_LOHI(SDNode *N);
- SDValue visitUMUL_LOHI(SDNode *N);
- SDValue visitSMULO(SDNode *N);
- SDValue visitUMULO(SDNode *N);
- SDValue visitSDIVREM(SDNode *N);
- SDValue visitUDIVREM(SDNode *N);
- SDValue visitAND(SDNode *N);
- SDValue visitOR(SDNode *N);
- SDValue visitXOR(SDNode *N);
- SDValue SimplifyVBinOp(SDNode *N);
- SDValue SimplifyVUnaryOp(SDNode *N);
- SDValue visitSHL(SDNode *N);
- SDValue visitSRA(SDNode *N);
- SDValue visitSRL(SDNode *N);
- SDValue visitCTLZ(SDNode *N);
- SDValue visitCTLZ_ZERO_UNDEF(SDNode *N);
- SDValue visitCTTZ(SDNode *N);
- SDValue visitCTTZ_ZERO_UNDEF(SDNode *N);
- SDValue visitCTPOP(SDNode *N);
- SDValue visitSELECT(SDNode *N);
- SDValue visitVSELECT(SDNode *N);
- SDValue visitSELECT_CC(SDNode *N);
- SDValue visitSETCC(SDNode *N);
- SDValue visitSIGN_EXTEND(SDNode *N);
- SDValue visitZERO_EXTEND(SDNode *N);
- SDValue visitANY_EXTEND(SDNode *N);
- SDValue visitSIGN_EXTEND_INREG(SDNode *N);
- SDValue visitTRUNCATE(SDNode *N);
- SDValue visitBITCAST(SDNode *N);
- SDValue visitBUILD_PAIR(SDNode *N);
- SDValue visitFADD(SDNode *N);
- SDValue visitFSUB(SDNode *N);
- SDValue visitFMUL(SDNode *N);
- SDValue visitFMA(SDNode *N);
- SDValue visitFDIV(SDNode *N);
- SDValue visitFREM(SDNode *N);
- SDValue visitFCOPYSIGN(SDNode *N);
- SDValue visitSINT_TO_FP(SDNode *N);
- SDValue visitUINT_TO_FP(SDNode *N);
- SDValue visitFP_TO_SINT(SDNode *N);
- SDValue visitFP_TO_UINT(SDNode *N);
- SDValue visitFP_ROUND(SDNode *N);
- SDValue visitFP_ROUND_INREG(SDNode *N);
- SDValue visitFP_EXTEND(SDNode *N);
- SDValue visitFNEG(SDNode *N);
- SDValue visitFABS(SDNode *N);
- SDValue visitFCEIL(SDNode *N);
- SDValue visitFTRUNC(SDNode *N);
- SDValue visitFFLOOR(SDNode *N);
- SDValue visitBRCOND(SDNode *N);
- SDValue visitBR_CC(SDNode *N);
- SDValue visitLOAD(SDNode *N);
- SDValue visitSTORE(SDNode *N);
- SDValue visitINSERT_VECTOR_ELT(SDNode *N);
- SDValue visitEXTRACT_VECTOR_ELT(SDNode *N);
- SDValue visitBUILD_VECTOR(SDNode *N);
- SDValue visitCONCAT_VECTORS(SDNode *N);
- SDValue visitEXTRACT_SUBVECTOR(SDNode *N);
- SDValue visitVECTOR_SHUFFLE(SDNode *N);
- SDValue XformToShuffleWithZero(SDNode *N);
- SDValue ReassociateOps(unsigned Opc, SDLoc DL, SDValue LHS, SDValue RHS);
- SDValue visitShiftByConstant(SDNode *N, unsigned Amt);
- bool SimplifySelectOps(SDNode *SELECT, SDValue LHS, SDValue RHS);
- SDValue SimplifyBinOpWithSameOpcodeHands(SDNode *N);
- SDValue SimplifySelect(SDLoc DL, SDValue N0, SDValue N1, SDValue N2);
- SDValue SimplifySelectCC(SDLoc DL, SDValue N0, SDValue N1, SDValue N2,
- SDValue N3, ISD::CondCode CC,
- bool NotExtCompare = false);
- SDValue SimplifySetCC(EVT VT, SDValue N0, SDValue N1, ISD::CondCode Cond,
- SDLoc DL, bool foldBooleans = true);
- SDValue SimplifyNodeWithTwoResults(SDNode *N, unsigned LoOp,
- unsigned HiOp);
- SDValue CombineConsecutiveLoads(SDNode *N, EVT VT);
- SDValue ConstantFoldBITCASTofBUILD_VECTOR(SDNode *, EVT);
- SDValue BuildSDIV(SDNode *N);
- SDValue BuildUDIV(SDNode *N);
- SDValue MatchBSwapHWordLow(SDNode *N, SDValue N0, SDValue N1,
- bool DemandHighBits = true);
- SDValue MatchBSwapHWord(SDNode *N, SDValue N0, SDValue N1);
- SDNode *MatchRotate(SDValue LHS, SDValue RHS, SDLoc DL);
- SDValue ReduceLoadWidth(SDNode *N);
- SDValue ReduceLoadOpStoreWidth(SDNode *N);
- SDValue TransformFPLoadStorePair(SDNode *N);
- SDValue reduceBuildVecExtToExtBuildVec(SDNode *N);
- SDValue reduceBuildVecConvertToConvertBuildVec(SDNode *N);
- SDValue GetDemandedBits(SDValue V, const APInt &Mask);
- /// GatherAllAliases - Walk up chain skipping non-aliasing memory nodes,
- /// looking for aliasing nodes and adding them to the Aliases vector.
- void GatherAllAliases(SDNode *N, SDValue OriginalChain,
- SmallVectorImpl<SDValue> &Aliases);
- /// isAlias - Return true if there is any possibility that the two addresses
- /// overlap.
- bool isAlias(SDValue Ptr1, int64_t Size1, bool IsVolatile1,
- const Value *SrcValue1, int SrcValueOffset1,
- unsigned SrcValueAlign1,
- const MDNode *TBAAInfo1,
- SDValue Ptr2, int64_t Size2, bool IsVolatile2,
- const Value *SrcValue2, int SrcValueOffset2,
- unsigned SrcValueAlign2,
- const MDNode *TBAAInfo2) const;
- /// isAlias - Return true if there is any possibility that the two addresses
- /// overlap.
- bool isAlias(LSBaseSDNode *Op0, LSBaseSDNode *Op1);
- /// FindAliasInfo - Extracts the relevant alias information from the memory
- /// node. Returns true if the operand was a load.
- bool FindAliasInfo(SDNode *N,
- SDValue &Ptr, int64_t &Size, bool &IsVolatile,
- const Value *&SrcValue, int &SrcValueOffset,
- unsigned &SrcValueAlignment,
- const MDNode *&TBAAInfo) const;
- /// FindBetterChain - Walk up chain skipping non-aliasing memory nodes,
- /// looking for a better chain (aliasing node.)
- SDValue FindBetterChain(SDNode *N, SDValue Chain);
- /// Merge consecutive store operations into a wide store.
- /// This optimization uses wide integers or vectors when possible.
- /// \return True if some memory operations were changed.
- bool MergeConsecutiveStores(StoreSDNode *N);
- public:
- DAGCombiner(SelectionDAG &D, AliasAnalysis &A, CodeGenOpt::Level OL)
- : DAG(D), TLI(D.getTargetLoweringInfo()), Level(BeforeLegalizeTypes),
- OptLevel(OL), LegalOperations(false), LegalTypes(false), AA(A) {
- AttributeSet FnAttrs =
- DAG.getMachineFunction().getFunction()->getAttributes();
- ForCodeSize =
- FnAttrs.hasAttribute(AttributeSet::FunctionIndex,
- Attribute::OptimizeForSize) ||
- FnAttrs.hasAttribute(AttributeSet::FunctionIndex, Attribute::MinSize);
- }
- /// Run - runs the dag combiner on all nodes in the work list
- void Run(CombineLevel AtLevel);
- SelectionDAG &getDAG() const { return DAG; }
- /// getShiftAmountTy - Returns a type large enough to hold any valid
- /// shift amount - before type legalization these can be huge.
- EVT getShiftAmountTy(EVT LHSTy) {
- assert(LHSTy.isInteger() && "Shift amount is not an integer type!");
- if (LHSTy.isVector())
- return LHSTy;
- return LegalTypes ? TLI.getScalarShiftAmountTy(LHSTy)
- : TLI.getPointerTy();
- }
- /// isTypeLegal - This method returns true if we are running before type
- /// legalization or if the specified VT is legal.
- bool isTypeLegal(const EVT &VT) {
- if (!LegalTypes) return true;
- return TLI.isTypeLegal(VT);
- }
- /// getSetCCResultType - Convenience wrapper around
- /// TargetLowering::getSetCCResultType
- EVT getSetCCResultType(EVT VT) const {
- return TLI.getSetCCResultType(*DAG.getContext(), VT);
- }
- };
- }
- namespace {
- /// WorkListRemover - This class is a DAGUpdateListener that removes any deleted
- /// nodes from the worklist.
- class WorkListRemover : public SelectionDAG::DAGUpdateListener {
- DAGCombiner &DC;
- public:
- explicit WorkListRemover(DAGCombiner &dc)
- : SelectionDAG::DAGUpdateListener(dc.getDAG()), DC(dc) {}
- virtual void NodeDeleted(SDNode *N, SDNode *E) {
- DC.removeFromWorkList(N);
- }
- };
- }
- //===----------------------------------------------------------------------===//
- // TargetLowering::DAGCombinerInfo implementation
- //===----------------------------------------------------------------------===//
- void TargetLowering::DAGCombinerInfo::AddToWorklist(SDNode *N) {
- ((DAGCombiner*)DC)->AddToWorkList(N);
- }
- void TargetLowering::DAGCombinerInfo::RemoveFromWorklist(SDNode *N) {
- ((DAGCombiner*)DC)->removeFromWorkList(N);
- }
- SDValue TargetLowering::DAGCombinerInfo::
- CombineTo(SDNode *N, const std::vector<SDValue> &To, bool AddTo) {
- return ((DAGCombiner*)DC)->CombineTo(N, &To[0], To.size(), AddTo);
- }
- SDValue TargetLowering::DAGCombinerInfo::
- CombineTo(SDNode *N, SDValue Res, bool AddTo) {
- return ((DAGCombiner*)DC)->CombineTo(N, Res, AddTo);
- }
- SDValue TargetLowering::DAGCombinerInfo::
- CombineTo(SDNode *N, SDValue Res0, SDValue Res1, bool AddTo) {
- return ((DAGCombiner*)DC)->CombineTo(N, Res0, Res1, AddTo);
- }
- void TargetLowering::DAGCombinerInfo::
- CommitTargetLoweringOpt(const TargetLowering::TargetLoweringOpt &TLO) {
- return ((DAGCombiner*)DC)->CommitTargetLoweringOpt(TLO);
- }
- //===----------------------------------------------------------------------===//
- // Helper Functions
- //===----------------------------------------------------------------------===//
- /// isNegatibleForFree - Return 1 if we can compute the negated form of the
- /// specified expression for the same cost as the expression itself, or 2 if we
- /// can compute the negated form more cheaply than the expression itself.
- static char isNegatibleForFree(SDValue Op, bool LegalOperations,
- const TargetLowering &TLI,
- const TargetOptions *Options,
- unsigned Depth = 0) {
- // fneg is removable even if it has multiple uses.
- if (Op.getOpcode() == ISD::FNEG) return 2;
- // Don't allow anything with multiple uses.
- if (!Op.hasOneUse()) return 0;
- // Don't recurse exponentially.
- if (Depth > 6) return 0;
- switch (Op.getOpcode()) {
- default: return false;
- case ISD::ConstantFP:
- // Don't invert constant FP values after legalize. The negated constant
- // isn't necessarily legal.
- return LegalOperations ? 0 : 1;
- case ISD::FADD:
- // FIXME: determine better conditions for this xform.
- if (!Options->UnsafeFPMath) return 0;
- // After operation legalization, it might not be legal to create new FSUBs.
- if (LegalOperations &&
- !TLI.isOperationLegalOrCustom(ISD::FSUB, Op.getValueType()))
- return 0;
- // fold (fneg (fadd A, B)) -> (fsub (fneg A), B)
- if (char V = isNegatibleForFree(Op.getOperand(0), LegalOperations, TLI,
- Options, Depth + 1))
- return V;
- // fold (fneg (fadd A, B)) -> (fsub (fneg B), A)
- return isNegatibleForFree(Op.getOperand(1), LegalOperations, TLI, Options,
- Depth + 1);
- case ISD::FSUB:
- // We can't turn -(A-B) into B-A when we honor signed zeros.
- if (!Options->UnsafeFPMath) return 0;
- // fold (fneg (fsub A, B)) -> (fsub B, A)
- return 1;
- case ISD::FMUL:
- case ISD::FDIV:
- if (Options->HonorSignDependentRoundingFPMath()) return 0;
- // fold (fneg (fmul X, Y)) -> (fmul (fneg X), Y) or (fmul X, (fneg Y))
- if (char V = isNegatibleForFree(Op.getOperand(0), LegalOperations, TLI,
- Options, Depth + 1))
- return V;
- return isNegatibleForFree(Op.getOperand(1), LegalOperations, TLI, Options,
- Depth + 1);
- case ISD::FP_EXTEND:
- case ISD::FP_ROUND:
- case ISD::FSIN:
- return isNegatibleForFree(Op.getOperand(0), LegalOperations, TLI, Options,
- Depth + 1);
- }
- }
- /// GetNegatedExpression - If isNegatibleForFree returns true, this function
- /// returns the newly negated expression.
- static SDValue GetNegatedExpression(SDValue Op, SelectionDAG &DAG,
- bool LegalOperations, unsigned Depth = 0) {
- // fneg is removable even if it has multiple uses.
- if (Op.getOpcode() == ISD::FNEG) return Op.getOperand(0);
- // Don't allow anything with multiple uses.
- assert(Op.hasOneUse() && "Unknown reuse!");
- assert(Depth <= 6 && "GetNegatedExpression doesn't match isNegatibleForFree");
- switch (Op.getOpcode()) {
- default: llvm_unreachable("Unknown code");
- case ISD::ConstantFP: {
- APFloat V = cast<ConstantFPSDNode>(Op)->getValueAPF();
- V.changeSign();
- return DAG.getConstantFP(V, Op.getValueType());
- }
- case ISD::FADD:
- // FIXME: determine better conditions for this xform.
- assert(DAG.getTarget().Options.UnsafeFPMath);
- // fold (fneg (fadd A, B)) -> (fsub (fneg A), B)
- if (isNegatibleForFree(Op.getOperand(0), LegalOperations,
- DAG.getTargetLoweringInfo(),
- &DAG.getTarget().Options, Depth+1))
- return DAG.getNode(ISD::FSUB, SDLoc(Op), Op.getValueType(),
- GetNegatedExpression(Op.getOperand(0), DAG,
- LegalOperations, Depth+1),
- Op.getOperand(1));
- // fold (fneg (fadd A, B)) -> (fsub (fneg B), A)
- return DAG.getNode(ISD::FSUB, SDLoc(Op), Op.getValueType(),
- GetNegatedExpression(Op.getOperand(1), DAG,
- LegalOperations, Depth+1),
- Op.getOperand(0));
- case ISD::FSUB:
- // We can't turn -(A-B) into B-A when we honor signed zeros.
- assert(DAG.getTarget().Options.UnsafeFPMath);
- // fold (fneg (fsub 0, B)) -> B
- if (ConstantFPSDNode *N0CFP = dyn_cast<ConstantFPSDNode>(Op.getOperand(0)))
- if (N0CFP->getValueAPF().isZero())
- return Op.getOperand(1);
- // fold (fneg (fsub A, B)) -> (fsub B, A)
- return DAG.getNode(ISD::FSUB, SDLoc(Op), Op.getValueType(),
- Op.getOperand(1), Op.getOperand(0));
- case ISD::FMUL:
- case ISD::FDIV:
- assert(!DAG.getTarget().Options.HonorSignDependentRoundingFPMath());
- // fold (fneg (fmul X, Y)) -> (fmul (fneg X), Y)
- if (isNegatibleForFree(Op.getOperand(0), LegalOperations,
- DAG.getTargetLoweringInfo(),
- &DAG.getTarget().Options, Depth+1))
- return DAG.getNode(Op.getOpcode(), SDLoc(Op), Op.getValueType(),
- GetNegatedExpression(Op.getOperand(0), DAG,
- LegalOperations, Depth+1),
- Op.getOperand(1));
- // fold (fneg (fmul X, Y)) -> (fmul X, (fneg Y))
- return DAG.getNode(Op.getOpcode(), SDLoc(Op), Op.getValueType(),
- Op.getOperand(0),
- GetNegatedExpression(Op.getOperand(1), DAG,
- LegalOperations, Depth+1));
- case ISD::FP_EXTEND:
- case ISD::FSIN:
- return DAG.getNode(Op.getOpcode(), SDLoc(Op), Op.getValueType(),
- GetNegatedExpression(Op.getOperand(0), DAG,
- LegalOperations, Depth+1));
- case ISD::FP_ROUND:
- return DAG.getNode(ISD::FP_ROUND, SDLoc(Op), Op.getValueType(),
- GetNegatedExpression(Op.getOperand(0), DAG,
- LegalOperations, Depth+1),
- Op.getOperand(1));
- }
- }
- // isSetCCEquivalent - Return true if this node is a setcc, or is a select_cc
- // that selects between the values 1 and 0, making it equivalent to a setcc.
- // Also, set the incoming LHS, RHS, and CC references to the appropriate
- // nodes based on the type of node we are checking. This simplifies life a
- // bit for the callers.
- static bool isSetCCEquivalent(SDValue N, SDValue &LHS, SDValue &RHS,
- SDValue &CC) {
- if (N.getOpcode() == ISD::SETCC) {
- LHS = N.getOperand(0);
- RHS = N.getOperand(1);
- CC = N.getOperand(2);
- return true;
- }
- if (N.getOpcode() == ISD::SELECT_CC &&
- N.getOperand(2).getOpcode() == ISD::Constant &&
- N.getOperand(3).getOpcode() == ISD::Constant &&
- cast<ConstantSDNode>(N.getOperand(2))->getAPIntValue() == 1 &&
- cast<ConstantSDNode>(N.getOperand(3))->isNullValue()) {
- LHS = N.getOperand(0);
- RHS = N.getOperand(1);
- CC = N.getOperand(4);
- return true;
- }
- return false;
- }
- // isOneUseSetCC - Return true if this is a SetCC-equivalent operation with only
- // one use. If this is true, it allows the users to invert the operation for
- // free when it is profitable to do so.
- static bool isOneUseSetCC(SDValue N) {
- SDValue N0, N1, N2;
- if (isSetCCEquivalent(N, N0, N1, N2) && N.getNode()->hasOneUse())
- return true;
- return false;
- }
- SDValue DAGCombiner::ReassociateOps(unsigned Opc, SDLoc DL,
- SDValue N0, SDValue N1) {
- EVT VT = N0.getValueType();
- if (N0.getOpcode() == Opc && isa<ConstantSDNode>(N0.getOperand(1))) {
- if (isa<ConstantSDNode>(N1)) {
- // reassoc. (op (op x, c1), c2) -> (op x, (op c1, c2))
- SDValue OpNode =
- DAG.FoldConstantArithmetic(Opc, VT,
- cast<ConstantSDNode>(N0.getOperand(1)),
- cast<ConstantSDNode>(N1));
- return DAG.getNode(Opc, DL, VT, N0.getOperand(0), OpNode);
- }
- if (N0.hasOneUse()) {
- // reassoc. (op (op x, c1), y) -> (op (op x, y), c1) iff x+c1 has one use
- SDValue OpNode = DAG.getNode(Opc, SDLoc(N0), VT,
- N0.getOperand(0), N1);
- AddToWorkList(OpNode.getNode());
- return DAG.getNode(Opc, DL, VT, OpNode, N0.getOperand(1));
- }
- }
- if (N1.getOpcode() == Opc && isa<ConstantSDNode>(N1.getOperand(1))) {
- if (isa<ConstantSDNode>(N0)) {
- // reassoc. (op c2, (op x, c1)) -> (op x, (op c1, c2))
- SDValue OpNode =
- DAG.FoldConstantArithmetic(Opc, VT,
- cast<ConstantSDNode>(N1.getOperand(1)),
- cast<ConstantSDNode>(N0));
- return DAG.getNode(Opc, DL, VT, N1.getOperand(0), OpNode);
- }
- if (N1.hasOneUse()) {
- // reassoc. (op y, (op x, c1)) -> (op (op x, y), c1) iff x+c1 has one use
- SDValue OpNode = DAG.getNode(Opc, SDLoc(N0), VT,
- N1.getOperand(0), N0);
- AddToWorkList(OpNode.getNode());
- return DAG.getNode(Opc, DL, VT, OpNode, N1.getOperand(1));
- }
- }
- return SDValue();
- }
- SDValue DAGCombiner::CombineTo(SDNode *N, const SDValue *To, unsigned NumTo,
- bool AddTo) {
- assert(N->getNumValues() == NumTo && "Broken CombineTo call!");
- ++NodesCombined;
- DEBUG(dbgs() << "\nReplacing.1 ";
- N->dump(&DAG);
- dbgs() << "\nWith: ";
- To[0].getNode()->dump(&DAG);
- dbgs() << " and " << NumTo-1 << " other values\n";
- for (unsigned i = 0, e = NumTo; i != e; ++i)
- assert((!To[i].getNode() ||
- N->getValueType(i) == To[i].getValueType()) &&
- "Cannot combine value to value of different type!"));
- WorkListRemover DeadNodes(*this);
- DAG.ReplaceAllUsesWith(N, To);
- if (AddTo) {
- // Push the new nodes and any users onto the worklist
- for (unsigned i = 0, e = NumTo; i != e; ++i) {
- if (To[i].getNode()) {
- AddToWorkList(To[i].getNode());
- AddUsersToWorkList(To[i].getNode());
- }
- }
- }
- // Finally, if the node is now dead, remove it from the graph. The node
- // may not be dead if the replacement process recursively simplified to
- // something else needing this node.
- if (N->use_empty()) {
- // Nodes can be reintroduced into the worklist. Make sure we do not
- // process a node that has been replaced.
- removeFromWorkList(N);
- // Finally, since the node is now dead, remove it from the graph.
- DAG.DeleteNode(N);
- }
- return SDValue(N, 0);
- }
- void DAGCombiner::
- CommitTargetLoweringOpt(const TargetLowering::TargetLoweringOpt &TLO) {
- // Replace all uses. If any nodes become isomorphic to other nodes and
- // are deleted, make sure to remove them from our worklist.
- WorkListRemover DeadNodes(*this);
- DAG.ReplaceAllUsesOfValueWith(TLO.Old, TLO.New);
- // Push the new node and any (possibly new) users onto the worklist.
- AddToWorkList(TLO.New.getNode());
- AddUsersToWorkList(TLO.New.getNode());
- // Finally, if the node is now dead, remove it from the graph. The node
- // may not be dead if the replacement process recursively simplified to
- // something else needing this node.
- if (TLO.Old.getNode()->use_empty()) {
- removeFromWorkList(TLO.Old.getNode());
- // If the operands of this node are only used by the node, they will now
- // be dead. Make sure to visit them first to delete dead nodes early.
- for (unsigned i = 0, e = TLO.Old.getNode()->getNumOperands(); i != e; ++i)
- if (TLO.Old.getNode()->getOperand(i).getNode()->hasOneUse())
- AddToWorkList(TLO.Old.getNode()->getOperand(i).getNode());
- DAG.DeleteNode(TLO.Old.getNode());
- }
- }
- /// SimplifyDemandedBits - Check the specified integer node value to see if
- /// it can be simplified or if things it uses can be simplified by bit
- /// propagation. If so, return true.
- bool DAGCombiner::SimplifyDemandedBits(SDValue Op, const APInt &Demanded) {
- TargetLowering::TargetLoweringOpt TLO(DAG, LegalTypes, LegalOperations);
- APInt KnownZero, KnownOne;
- if (!TLI.SimplifyDemandedBits(Op, Demanded, KnownZero, KnownOne, TLO))
- return false;
- // Revisit the node.
- AddToWorkList(Op.getNode());
- // Replace the old value with the new one.
- ++NodesCombined;
- DEBUG(dbgs() << "\nReplacing.2 ";
- TLO.Old.getNode()->dump(&DAG);
- dbgs() << "\nWith: ";
- TLO.New.getNode()->dump(&DAG);
- dbgs() << '\n');
- CommitTargetLoweringOpt(TLO);
- return true;
- }
- void DAGCombiner::ReplaceLoadWithPromotedLoad(SDNode *Load, SDNode *ExtLoad) {
- SDLoc dl(Load);
- EVT VT = Load->getValueType(0);
- SDValue Trunc = DAG.getNode(ISD::TRUNCATE, dl, VT, SDValue(ExtLoad, 0));
- DEBUG(dbgs() << "\nReplacing.9 ";
- Load->dump(&DAG);
- dbgs() << "\nWith: ";
- Trunc.getNode()->dump(&DAG);
- dbgs() << '\n');
- WorkListRemover DeadNodes(*this);
- DAG.ReplaceAllUsesOfValueWith(SDValue(Load, 0), Trunc);
- DAG.ReplaceAllUsesOfValueWith(SDValue(Load, 1), SDValue(ExtLoad, 1));
- removeFromWorkList(Load);
- DAG.DeleteNode(Load);
- AddToWorkList(Trunc.getNode());
- }
- SDValue DAGCombiner::PromoteOperand(SDValue Op, EVT PVT, bool &Replace) {
- Replace = false;
- SDLoc dl(Op);
- if (LoadSDNode *LD = dyn_cast<LoadSDNode>(Op)) {
- EVT MemVT = LD->getMemoryVT();
- ISD::LoadExtType ExtType = ISD::isNON_EXTLoad(LD)
- ? (TLI.isLoadExtLegal(ISD::ZEXTLOAD, MemVT) ? ISD::ZEXTLOAD
- : ISD::EXTLOAD)
- : LD->getExtensionType();
- Replace = true;
- return DAG.getExtLoad(ExtType, dl, PVT,
- LD->getChain(), LD->getBasePtr(),
- MemVT, LD->getMemOperand());
- }
- unsigned Opc = Op.getOpcode();
- switch (Opc) {
- default: break;
- case ISD::AssertSext:
- return DAG.getNode(ISD::AssertSext, dl, PVT,
- SExtPromoteOperand(Op.getOperand(0), PVT),
- Op.getOperand(1));
- case ISD::AssertZext:
- return DAG.getNode(ISD::AssertZext, dl, PVT,
- ZExtPromoteOperand(Op.getOperand(0), PVT),
- Op.getOperand(1));
- case ISD::Constant: {
- unsigned ExtOpc =
- Op.getValueType().isByteSized() ? ISD::SIGN_EXTEND : ISD::ZERO_EXTEND;
- return DAG.getNode(ExtOpc, dl, PVT, Op);
- }
- }
- if (!TLI.isOperationLegal(ISD::ANY_EXTEND, PVT))
- return SDValue();
- return DAG.getNode(ISD::ANY_EXTEND, dl, PVT, Op);
- }
- SDValue DAGCombiner::SExtPromoteOperand(SDValue Op, EVT PVT) {
- if (!TLI.isOperationLegal(ISD::SIGN_EXTEND_INREG, PVT))
- return SDValue();
- EVT OldVT = Op.getValueType();
- SDLoc dl(Op);
- bool Replace = false;
- SDValue NewOp = PromoteOperand(Op, PVT, Replace);
- if (NewOp.getNode() == 0)
- return SDValue();
- AddToWorkList(NewOp.getNode());
- if (Replace)
- ReplaceLoadWithPromotedLoad(Op.getNode(), NewOp.getNode());
- return DAG.getNode(ISD::SIGN_EXTEND_INREG, dl, NewOp.getValueType(), NewOp,
- DAG.getValueType(OldVT));
- }
- SDValue DAGCombiner::ZExtPromoteOperand(SDValue Op, EVT PVT) {
- EVT OldVT = Op.getValueType();
- SDLoc dl(Op);
- bool Replace = false;
- SDValue NewOp = PromoteOperand(Op, PVT, Replace);
- if (NewOp.getNode() == 0)
- return SDValue();
- AddToWorkList(NewOp.getNode());
- if (Replace)
- ReplaceLoadWithPromotedLoad(Op.getNode(), NewOp.getNode());
- return DAG.getZeroExtendInReg(NewOp, dl, OldVT);
- }
- /// PromoteIntBinOp - Promote the specified integer binary operation if the
- /// target indicates it is beneficial. e.g. On x86, it's usually better to
- /// promote i16 operations to i32 since i16 instructions are longer.
- SDValue DAGCombiner::PromoteIntBinOp(SDValue Op) {
- if (!LegalOperations)
- return SDValue();
- EVT VT = Op.getValueType();
- if (VT.isVector() || !VT.isInteger())
- return SDValue();
- // If operation type is 'undesirable', e.g. i16 on x86, consider
- // promoting it.
- unsigned Opc = Op.getOpcode();
- if (TLI.isTypeDesirableForOp(Opc, VT))
- return SDValue();
- EVT PVT = VT;
- // Consult target whether it is a good idea to promote this operation and
- // what's the right type to promote it to.
- if (TLI.IsDesirableToPromoteOp(Op, PVT)) {
- assert(PVT != VT && "Don't know what type to promote to!");
- bool Replace0 = false;
- SDValue N0 = Op.getOperand(0);
- SDValue NN0 = PromoteOperand(N0, PVT, Replace0);
- if (NN0.getNode() == 0)
- return SDValue();
- bool Replace1 = false;
- SDValue N1 = Op.getOperand(1);
- SDValue NN1;
- if (N0 == N1)
- NN1 = NN0;
- else {
- NN1 = PromoteOperand(N1, PVT, Replace1);
- if (NN1.getNode() == 0)
- return SDValue();
- }
- AddToWorkList(NN0.getNode());
- if (NN1.getNode())
- AddToWorkList(NN1.getNode());
- if (Replace0)
- ReplaceLoadWithPromotedLoad(N0.getNode(), NN0.getNode());
- if (Replace1)
- ReplaceLoadWithPromotedLoad(N1.getNode(), NN1.getNode());
- DEBUG(dbgs() << "\nPromoting ";
- Op.getNode()->dump(&DAG));
- SDLoc dl(Op);
- return DAG.getNode(ISD::TRUNCATE, dl, VT,
- DAG.getNode(Opc, dl, PVT, NN0, NN1));
- }
- return SDValue();
- }
- /// PromoteIntShiftOp - Promote the specified integer shift operation if the
- /// target indicates it is beneficial. e.g. On x86, it's usually better to
- /// promote i16 operations to i32 since i16 instructions are longer.
- SDValue DAGCombiner::PromoteIntShiftOp(SDValue Op) {
- if (!LegalOperations)
- return SDValue();
- EVT VT = Op.getValueType();
- if (VT.isVector() || !VT.isInteger())
- return SDValue();
- // If operation type is 'undesirable', e.g. i16 on x86, consider
- // promoting it.
- unsigned Opc = Op.getOpcode();
- if (TLI.isTypeDesirableForOp(Opc, VT))
- return SDValue();
- EVT PVT = VT;
- // Consult target whether it is a good idea to promote this operation and
- // what's the right type to promote it to.
- if (TLI.IsDesirableToPromoteOp(Op, PVT)) {
- assert(PVT != VT && "Don't know what type to promote to!");
- bool Replace = false;
- SDValue N0 = Op.getOperand(0);
- if (Opc == ISD::SRA)
- N0 = SExtPromoteOperand(Op.getOperand(0), PVT);
- else if (Opc == ISD::SRL)
- N0 = ZExtPromoteOperand(Op.getOperand(0), PVT);
- else
- N0 = PromoteOperand(N0, PVT, Replace);
- if (N0.getNode() == 0)
- return SDValue();
- AddToWorkList(N0.getNode());
- if (Replace)
- ReplaceLoadWithPromotedLoad(Op.getOperand(0).getNode(), N0.getNode());
- DEBUG(dbgs() << "\nPromoting ";
- Op.getNode()->dump(&DAG));
- SDLoc dl(Op);
- return DAG.getNode(ISD::TRUNCATE, dl, VT,
- DAG.getNode(Opc, dl, PVT, N0, Op.getOperand(1)));
- }
- return SDValue();
- }
- SDValue DAGCombiner::PromoteExtend(SDValue Op) {
- if (!LegalOperations)
- return SDValue();
- EVT VT = Op.getValueType();
- if (VT.isVector() || !VT.isInteger())
- return SDValue();
- // If operation type is 'undesirable', e.g. i16 on x86, consider
- // promoting it.
- unsigned Opc = Op.getOpcode();
- if (TLI.isTypeDesirableForOp(Opc, VT))
- return SDValue();
- EVT PVT = VT;
- // Consult target whether it is a good idea to promote this operation and
- // what's the right type to promote it to.
- if (TLI.IsDesirableToPromoteOp(Op, PVT)) {
- assert(PVT != VT && "Don't know what type to promote to!");
- // fold (aext (aext x)) -> (aext x)
- // fold (aext (zext x)) -> (zext x)
- // fold (aext (sext x)) -> (sext x)
- DEBUG(dbgs() << "\nPromoting ";
- Op.getNode()->dump(&DAG));
- return DAG.getNode(Op.getOpcode(), SDLoc(Op), VT, Op.getOperand(0));
- }
- return SDValue();
- }
- bool DAGCombiner::PromoteLoad(SDValue Op) {
- if (!LegalOperations)
- return false;
- EVT VT = Op.getValueType();
- if (VT.isVector() || !VT.isInteger())
- return false;
- // If operation type is 'undesirable', e.g. i16 on x86, consider
- // promoting it.
- unsigned Opc = Op.getOpcode();
- if (TLI.isTypeDesirableForOp(Opc, VT))
- return false;
- EVT PVT = VT;
- // Consult target whether it is a good idea to promote this operation and
- // what's the right type to promote it to.
- if (TLI.IsDesirableToPromoteOp(Op, PVT)) {
- assert(PVT != VT && "Don't know what type to promote to!");
- SDLoc dl(Op);
- SDNode *N = Op.getNode();
- LoadSDNode *LD = cast<LoadSDNode>(N);
- EVT MemVT = LD->getMemoryVT();
- ISD::LoadExtType ExtType = ISD::isNON_EXTLoad(LD)
- ? (TLI.isLoadExtLegal(ISD::ZEXTLOAD, MemVT) ? ISD::ZEXTLOAD
- : ISD::EXTLOAD)
- : LD->getExtensionType();
- SDValue NewLD = DAG.getExtLoad(ExtType, dl, PVT,
- LD->getChain(), LD->getBasePtr(),
- MemVT, LD->getMemOperand());
- SDValue Result = DAG.getNode(ISD::TRUNCATE, dl, VT, NewLD);
- DEBUG(dbgs() << "\nPromoting ";
- N->dump(&DAG);
- dbgs() << "\nTo: ";
- Result.getNode()->dump(&DAG);
- dbgs() << '\n');
- WorkListRemover DeadNodes(*this);
- DAG.ReplaceAllUsesOfValueWith(SDValue(N, 0), Result);
- DAG.ReplaceAllUsesOfValueWith(SDValue(N, 1), NewLD.getValue(1));
- removeFromWorkList(N);
- DAG.DeleteNode(N);
- AddToWorkList(Result.getNode());
- return true;
- }
- return false;
- }
- //===----------------------------------------------------------------------===//
- // Main DAG Combiner implementation
- //===----------------------------------------------------------------------===//
- void DAGCombiner::Run(CombineLevel AtLevel) {
- // set the instance variables, so that the various visit routines may use it.
- Level = AtLevel;
- LegalOperations = Level >= AfterLegalizeVectorOps;
- LegalTypes = Level >= AfterLegalizeTypes;
- // Add all the dag nodes to the worklist.
- for (SelectionDAG::allnodes_iterator I = DAG.allnodes_begin(),
- E = DAG.allnodes_end(); I != E; ++I)
- AddToWorkList(I);
- // Create a dummy node (which is not added to allnodes), that adds a reference
- // to the root node, preventing it from being deleted, and tracking any
- // changes of the root.
- HandleSDNode Dummy(DAG.getRoot());
- // The root of the dag may dangle to deleted nodes until the dag combiner is
- // done. Set it to null to avoid confusion.
- DAG.setRoot(SDValue());
- // while the worklist isn't empty, find a node and
- // try and combine it.
- while (!WorkListContents.empty()) {
- SDNode *N;
- // The WorkListOrder holds the SDNodes in order, but it may contain
- // duplicates.
- // In order to avoid a linear scan, we use a set (O(log N)) to hold what the
- // worklist *should* contain, and check the node we want to visit is should
- // actually be visited.
- do {
- N = WorkListOrder.pop_back_val();
- } while (!WorkListContents.erase(N));
- // If N has no uses, it is dead. Make sure to revisit all N's operands once
- // N is deleted from the DAG, since they too may now be dead or may have a
- // reduced number of uses, allowing other xforms.
- if (N->use_empty() && N != &Dummy) {
- for (unsigned i = 0, e = N->getNumOperands(); i != e; ++i)
- AddToWorkList(N->getOperand(i).getNode());
- DAG.DeleteNode(N);
- continue;
- }
- SDValue RV = combine(N);
- if (RV.getNode() == 0)
- continue;
- ++NodesCombined;
- // If we get back the same node we passed in, rather than a new node or
- // zero, we know that the node must have defined multiple values and
- // CombineTo was used. Since CombineTo takes care of the worklist
- // mechanics for us, we have no work to do in this case.
- if (RV.getNode() == N)
- continue;
- assert(N->getOpcode() != ISD::DELETED_NODE &&
- RV.getNode()->getOpcode() != ISD::DELETED_NODE &&
- "Node was deleted but visit returned new node!");
- DEBUG(dbgs() << "\nReplacing.3 ";
- N->dump(&DAG);
- dbgs() << "\nWith: ";
- RV.getNode()->dump(&DAG);
- dbgs() << '\n');
- // Transfer debug value.
- DAG.TransferDbgValues(SDValue(N, 0), RV);
- WorkListRemover DeadNodes(*this);
- if (N->getNumValues() == RV.getNode()->getNumValues())
- DAG.ReplaceAllUsesWith(N, RV.getNode());
- else {
- assert(N->getValueType(0) == RV.getValueType() &&
- N->getNumValues() == 1 && "Type mismatch");
- SDValue OpV = RV;
- DAG.ReplaceAllUsesWith(N, &OpV);
- }
- // Push the new node and any users onto the worklist
- AddToWorkList(RV.getNode());
- AddUsersToWorkList(RV.getNode());
- // Add any uses of the old node to the worklist in case this node is the
- // last one that uses them. They may become dead after this node is
- // deleted.
- for (unsigned i = 0, e = N->getNumOperands(); i != e; ++i)
- AddToWorkList(N->getOperand(i).getNode());
- // Finally, if the node is now dead, remove it from the graph. The node
- // may not be dead if the replacement process recursively simplified to
- // something else needing this node.
- if (N->use_empty()) {
- // Nodes can be reintroduced into the worklist. Make sure we do not
- // process a node that has been replaced.
- removeFromWorkList(N);
- // Finally, since the node is now dead, remove it from the graph.
- DAG.DeleteNode(N);
- }
- }
- // If the root changed (e.g. it was a dead load, update the root).
- DAG.setRoot(Dummy.getValue());
- DAG.RemoveDeadNodes();
- }
- SDValue DAGCombiner::visit(SDNode *N) {
- switch (N->getOpcode()) {
- default: break;
- case ISD::TokenFactor: return visitTokenFactor(N);
- case ISD::MERGE_VALUES: return visitMERGE_VALUES(N);
- case ISD::ADD: return visitADD(N);
- case ISD::SUB: return visitSUB(N);
- case ISD::ADDC: return visitADDC(N);
- case ISD::SUBC: return visitSUBC(N);
- case ISD::ADDE: return visitADDE(N);
- case ISD::SUBE: return visitSUBE(N);
- case ISD::MUL: return visitMUL(N);
- case ISD::SDIV: return visitSDIV(N);
- case ISD::UDIV: return visitUDIV(N);
- case ISD::SREM: return visitSREM(N);
- case ISD::UREM: return visitUREM(N);
- case ISD::MULHU: return visitMULHU(N);
- case ISD::MULHS: return visitMULHS(N);
- case ISD::SMUL_LOHI: return visitSMUL_LOHI(N);
- case ISD::UMUL_LOHI: return visitUMUL_LOHI(N);
- case ISD::SMULO: return visitSMULO(N);
- case ISD::UMULO: return visitUMULO(N);
- case ISD::SDIVREM: return visitSDIVREM(N);
- case ISD::UDIVREM: return visitUDIVREM(N);
- case ISD::AND: return visitAND(N);
- case ISD::OR: return visitOR(N);
- case ISD::XOR: return visitXOR(N);
- case ISD::SHL: return visitSHL(N);
- case ISD::SRA: return visitSRA(N);
- case ISD::SRL: return visitSRL(N);
- case ISD::CTLZ: return visitCTLZ(N);
- case ISD::CTLZ_ZERO_UNDEF: return visitCTLZ_ZERO_UNDEF(N);
- case ISD::CTTZ: return visitCTTZ(N);
- case ISD::CTTZ_ZERO_UNDEF: return visitCTTZ_ZERO_UNDEF(N);
- case ISD::CTPOP: return visitCTPOP(N);
- case ISD::SELECT: return visitSELECT(N);
- case ISD::VSELECT: return visitVSELECT(N);
- case ISD::SELECT_CC: return visitSELECT_CC(N);
- case ISD::SETCC: return visitSETCC(N);
- case ISD::SIGN_EXTEND: return visitSIGN_EXTEND(N);
- case ISD::ZERO_EXTEND: return visitZERO_EXTEND(N);
- case ISD::ANY_EXTEND: return visitANY_EXTEND(N);
- case ISD::SIGN_EXTEND_INREG: return visitSIGN_EXTEND_INREG(N);
- case ISD::TRUNCATE: return visitTRUNCATE(N);
- case ISD::BITCAST: return visitBITCAST(N);
- case ISD::BUILD_PAIR: return visitBUILD_PAIR(N);
- case ISD::FADD: return visitFADD(N);
- case ISD::FSUB: return visitFSUB(N);
- case ISD::FMUL: return visitFMUL(N);
- case ISD::FMA: return visitFMA(N);
- case ISD::FDIV: return visitFDIV(N);
- case ISD::FREM: return visitFREM(N);
- case ISD::FCOPYSIGN: return visitFCOPYSIGN(N);
- case ISD::SINT_TO_FP: return visitSINT_TO_FP(N);
- case ISD::UINT_TO_FP: return visitUINT_TO_FP(N);
- case ISD::FP_TO_SINT: return visitFP_TO_SINT(N);
- case ISD::FP_TO_UINT: return visitFP_TO_UINT(N);
- case ISD::FP_ROUND: return visitFP_ROUND(N);
- case ISD::FP_ROUND_INREG: return visitFP_ROUND_INREG(N);
- case ISD::FP_EXTEND: return visitFP_EXTEND(N);
- case ISD::FNEG: return visitFNEG(N);
- case ISD::FABS: return visitFABS(N);
- case ISD::FFLOOR: return visitFFLOOR(N);
- case ISD::FCEIL: return visitFCEIL(N);
- case ISD::FTRUNC: return visitFTRUNC(N);
- case ISD::BRCOND: return visitBRCOND(N);
- case ISD::BR_CC: return visitBR_CC(N);
- case ISD::LOAD: return visitLOAD(N);
- case ISD::STORE: return visitSTORE(N);
- case ISD::INSERT_VECTOR_ELT: return visitINSERT_VECTOR_ELT(N);
- case ISD::EXTRACT_VECTOR_ELT: return visitEXTRACT_VECTOR_ELT(N);
- case ISD::BUILD_VECTOR: return visitBUILD_VECTOR(N);
- case ISD::CONCAT_VECTORS: return visitCONCAT_VECTORS(N);
- case ISD::EXTRACT_SUBVECTOR: return visitEXTRACT_SUBVECTOR(N);
- case ISD::VECTOR_SHUFFLE: return visitVECTOR_SHUFFLE(N);
- }
- return SDValue();
- }
- SDValue DAGCombiner::combine(SDNode *N) {
- SDValue RV = visit(N);
- // If nothing happened, try a target-specific DAG combine.
- if (RV.getNode() == 0) {
- assert(N->getOpcode() != ISD::DELETED_NODE &&
- "Node was deleted but visit returned NULL!");
- if (N->getOpcode() >= ISD::BUILTIN_OP_END ||
- TLI.hasTargetDAGCombine((ISD::NodeType)N->getOpcode())) {
- // Expose the DAG combiner to the target combiner impls.
- TargetLowering::DAGCombinerInfo
- DagCombineInfo(DAG, Level, false, this);
- RV = TLI.PerformDAGCombine(N, DagCombineInfo);
- }
- }
- // If nothing happened still, try promoting the operation.
- if (RV.getNode() == 0) {
- switch (N->getOpcode()) {
- default: break;
- case ISD::ADD:
- case ISD::SUB:
- case ISD::MUL:
- case ISD::AND:
- case ISD::OR:
- case ISD::XOR:
- RV = PromoteIntBinOp(SDValue(N, 0));
- break;
- case ISD::SHL:
- case ISD::SRA:
- case ISD::SRL:
- RV = PromoteIntShiftOp(SDValue(N, 0));
- break;
- case ISD::SIGN_EXTEND:
- case ISD::ZERO_EXTEND:
- case ISD::ANY_EXTEND:
- RV = PromoteExtend(SDValue(N, 0));
- break;
- case ISD::LOAD:
- if (PromoteLoad(SDValue(N, 0)))
- RV = SDValue(N, 0);
- break;
- }
- }
- // If N is a commutative binary node, try commuting it to enable more
- // sdisel CSE.
- if (RV.getNode() == 0 &&
- SelectionDAG::isCommutativeBinOp(N->getOpcode()) &&
- N->getNumValues() == 1) {
- SDValue N0 = N->getOperand(0);
- SDValue N1 = N->getOperand(1);
- // Constant operands are canonicalized to RHS.
- if (isa<ConstantSDNode>(N0) || !isa<ConstantSDNode>(N1)) {
- SDValue Ops[] = { N1, N0 };
- SDNode *CSENode = DAG.getNodeIfExists(N->getOpcode(), N->getVTList(),
- Ops, 2);
- if (CSENode)
- return SDValue(CSENode, 0);
- }
- }
- return RV;
- }
- /// getInputChainForNode - Given a node, return its input chain if it has one,
- /// otherwise return a null sd operand.
- static SDValue getInputChainForNode(SDNode *N) {
- if (unsigned NumOps = N->getNumOperands()) {
- if (N->getOperand(0).getValueType() == MVT::Other)
- return N->getOperand(0);
- if (N->getOperand(NumOps-1).getValueType() == MVT::Other)
- return N->getOperand(NumOps-1);
- for (unsigned i = 1; i < NumOps-1; ++i)
- if (N->getOperand(i).getValueType() == MVT::Other)
- return N->getOperand(i);
- }
- return SDValue();
- }
- SDValue DAGCombiner::visitTokenFactor(SDNode *N) {
- // If N has two operands, where one has an input chain equal to the other,
- // the 'other' chain is redundant.
- if (N->getNumOperands() == 2) {
- if (getInputChainForNode(N->getOperand(0).getNode()) == N->getOperand(1))
- return N->getOperand(0);
- if (getInputChainForNode(N->getOperand(1).getNode()) == N->getOperand(0))
- return N->getOperand(1);
- }
- SmallVector<SDNode *, 8> TFs; // List of token factors to visit.
- SmallVector<SDValue, 8> Ops; // Ops for replacing token factor.
- SmallPtrSet<SDNode*, 16> SeenOps;
- bool Changed = false; // If we should replace this token factor.
- // Start out with this token factor.
- TFs.push_back(N);
- // Iterate through token factors. The TFs grows when new token factors are
- // encountered.
- for (unsigned i = 0; i < TFs.size(); ++i) {
- SDNode *TF = TFs[i];
- // Check each of the operands.
- for (unsigned i = 0, ie = TF->getNumOperands(); i != ie; ++i) {
- SDValue Op = TF->getOperand(i);
- switch (Op.getOpcode()) {
- case ISD::EntryToken:
- // Entry tokens don't need to be added to the list. They are
- // rededundant.
- Changed = true;
- break;
- case ISD::TokenFactor:
- if (Op.hasOneUse() &&
- std::find(TFs.begin(), TFs.end(), Op.getNode()) == TFs.end()) {
- // Q…