/3rd_party/llvm/lib/CodeGen/SelectionDAG/DAGCombiner.cpp
C++ | 11051 lines | 7632 code | 1358 blank | 2061 comment | 3191 complexity | 9168f24fe989eb6a6817a65e823725b7 MD5 | raw file
Possible License(s): LGPL-2.1, BSD-3-Clause, JSON, MPL-2.0-no-copyleft-exception, GPL-2.0, GPL-3.0, LGPL-3.0, BSD-2-Clause
Large files files are truncated, but you can click here to view the full file
1//===-- DAGCombiner.cpp - Implement a DAG node combiner -------------------===// 2// 3// The LLVM Compiler Infrastructure 4// 5// This file is distributed under the University of Illinois Open Source 6// License. See LICENSE.TXT for details. 7// 8//===----------------------------------------------------------------------===// 9// 10// This pass combines dag nodes to form fewer, simpler DAG nodes. It can be run 11// both before and after the DAG is legalized. 12// 13// This pass is not a substitute for the LLVM IR instcombine pass. This pass is 14// primarily intended to handle simplification opportunities that are implicit 15// in the LLVM IR and exposed by the various codegen lowering phases. 16// 17//===----------------------------------------------------------------------===// 18 19#define DEBUG_TYPE "dagcombine" 20#include "llvm/CodeGen/SelectionDAG.h" 21#include "llvm/ADT/SmallPtrSet.h" 22#include "llvm/ADT/Statistic.h" 23#include "llvm/Analysis/AliasAnalysis.h" 24#include "llvm/CodeGen/MachineFrameInfo.h" 25#include "llvm/CodeGen/MachineFunction.h" 26#include "llvm/IR/DataLayout.h" 27#include "llvm/IR/DerivedTypes.h" 28#include "llvm/IR/Function.h" 29#include "llvm/IR/LLVMContext.h" 30#include "llvm/Support/CommandLine.h" 31#include "llvm/Support/Debug.h" 32#include "llvm/Support/ErrorHandling.h" 33#include "llvm/Support/MathExtras.h" 34#include "llvm/Support/raw_ostream.h" 35#include "llvm/Target/TargetLowering.h" 36#include "llvm/Target/TargetMachine.h" 37#include "llvm/Target/TargetOptions.h" 38#include "llvm/Target/TargetRegisterInfo.h" 39#include "llvm/Target/TargetSubtargetInfo.h" 40#include <algorithm> 41using namespace llvm; 42 43STATISTIC(NodesCombined , "Number of dag nodes combined"); 44STATISTIC(PreIndexedNodes , "Number of pre-indexed nodes created"); 45STATISTIC(PostIndexedNodes, "Number of post-indexed nodes created"); 46STATISTIC(OpsNarrowed , "Number of load/op/store narrowed"); 47STATISTIC(LdStFP2Int , "Number of fp load/store pairs transformed to int"); 48STATISTIC(SlicedLoads, "Number of load sliced"); 49 50namespace { 51 static cl::opt<bool> 52 CombinerAA("combiner-alias-analysis", cl::Hidden, 53 cl::desc("Turn on alias analysis during testing")); 54 55 static cl::opt<bool> 56 CombinerGlobalAA("combiner-global-alias-analysis", cl::Hidden, 57 cl::desc("Include global information in alias analysis")); 58 59 /// Hidden option to stress test load slicing, i.e., when this option 60 /// is enabled, load slicing bypasses most of its profitability guards. 61 static cl::opt<bool> 62 StressLoadSlicing("combiner-stress-load-slicing", cl::Hidden, 63 cl::desc("Bypass the profitability model of load " 64 "slicing"), 65 cl::init(false)); 66 67//------------------------------ DAGCombiner ---------------------------------// 68 69 class DAGCombiner { 70 SelectionDAG &DAG; 71 const TargetLowering &TLI; 72 CombineLevel Level; 73 CodeGenOpt::Level OptLevel; 74 bool LegalOperations; 75 bool LegalTypes; 76 bool ForCodeSize; 77 78 // Worklist of all of the nodes that need to be simplified. 79 // 80 // This has the semantics that when adding to the worklist, 81 // the item added must be next to be processed. It should 82 // also only appear once. The naive approach to this takes 83 // linear time. 84 // 85 // To reduce the insert/remove time to logarithmic, we use 86 // a set and a vector to maintain our worklist. 87 // 88 // The set contains the items on the worklist, but does not 89 // maintain the order they should be visited. 90 // 91 // The vector maintains the order nodes should be visited, but may 92 // contain duplicate or removed nodes. When choosing a node to 93 // visit, we pop off the order stack until we find an item that is 94 // also in the contents set. All operations are O(log N). 95 SmallPtrSet<SDNode*, 64> WorkListContents; 96 SmallVector<SDNode*, 64> WorkListOrder; 97 98 // AA - Used for DAG load/store alias analysis. 99 AliasAnalysis &AA; 100 101 /// AddUsersToWorkList - When an instruction is simplified, add all users of 102 /// the instruction to the work lists because they might get more simplified 103 /// now. 104 /// 105 void AddUsersToWorkList(SDNode *N) { 106 for (SDNode::use_iterator UI = N->use_begin(), UE = N->use_end(); 107 UI != UE; ++UI) 108 AddToWorkList(*UI); 109 } 110 111 /// visit - call the node-specific routine that knows how to fold each 112 /// particular type of node. 113 SDValue visit(SDNode *N); 114 115 public: 116 /// AddToWorkList - Add to the work list making sure its instance is at the 117 /// back (next to be processed.) 118 void AddToWorkList(SDNode *N) { 119 WorkListContents.insert(N); 120 WorkListOrder.push_back(N); 121 } 122 123 /// removeFromWorkList - remove all instances of N from the worklist. 124 /// 125 void removeFromWorkList(SDNode *N) { 126 WorkListContents.erase(N); 127 } 128 129 SDValue CombineTo(SDNode *N, const SDValue *To, unsigned NumTo, 130 bool AddTo = true); 131 132 SDValue CombineTo(SDNode *N, SDValue Res, bool AddTo = true) { 133 return CombineTo(N, &Res, 1, AddTo); 134 } 135 136 SDValue CombineTo(SDNode *N, SDValue Res0, SDValue Res1, 137 bool AddTo = true) { 138 SDValue To[] = { Res0, Res1 }; 139 return CombineTo(N, To, 2, AddTo); 140 } 141 142 void CommitTargetLoweringOpt(const TargetLowering::TargetLoweringOpt &TLO); 143 144 private: 145 146 /// SimplifyDemandedBits - Check the specified integer node value to see if 147 /// it can be simplified or if things it uses can be simplified by bit 148 /// propagation. If so, return true. 149 bool SimplifyDemandedBits(SDValue Op) { 150 unsigned BitWidth = Op.getValueType().getScalarType().getSizeInBits(); 151 APInt Demanded = APInt::getAllOnesValue(BitWidth); 152 return SimplifyDemandedBits(Op, Demanded); 153 } 154 155 bool SimplifyDemandedBits(SDValue Op, const APInt &Demanded); 156 157 bool CombineToPreIndexedLoadStore(SDNode *N); 158 bool CombineToPostIndexedLoadStore(SDNode *N); 159 bool SliceUpLoad(SDNode *N); 160 161 void ReplaceLoadWithPromotedLoad(SDNode *Load, SDNode *ExtLoad); 162 SDValue PromoteOperand(SDValue Op, EVT PVT, bool &Replace); 163 SDValue SExtPromoteOperand(SDValue Op, EVT PVT); 164 SDValue ZExtPromoteOperand(SDValue Op, EVT PVT); 165 SDValue PromoteIntBinOp(SDValue Op); 166 SDValue PromoteIntShiftOp(SDValue Op); 167 SDValue PromoteExtend(SDValue Op); 168 bool PromoteLoad(SDValue Op); 169 170 void ExtendSetCCUses(const SmallVectorImpl<SDNode *> &SetCCs, 171 SDValue Trunc, SDValue ExtLoad, SDLoc DL, 172 ISD::NodeType ExtType); 173 174 /// combine - call the node-specific routine that knows how to fold each 175 /// particular type of node. If that doesn't do anything, try the 176 /// target-specific DAG combines. 177 SDValue combine(SDNode *N); 178 179 // Visitation implementation - Implement dag node combining for different 180 // node types. The semantics are as follows: 181 // Return Value: 182 // SDValue.getNode() == 0 - No change was made 183 // SDValue.getNode() == N - N was replaced, is dead and has been handled. 184 // otherwise - N should be replaced by the returned Operand. 185 // 186 SDValue visitTokenFactor(SDNode *N); 187 SDValue visitMERGE_VALUES(SDNode *N); 188 SDValue visitADD(SDNode *N); 189 SDValue visitSUB(SDNode *N); 190 SDValue visitADDC(SDNode *N); 191 SDValue visitSUBC(SDNode *N); 192 SDValue visitADDE(SDNode *N); 193 SDValue visitSUBE(SDNode *N); 194 SDValue visitMUL(SDNode *N); 195 SDValue visitSDIV(SDNode *N); 196 SDValue visitUDIV(SDNode *N); 197 SDValue visitSREM(SDNode *N); 198 SDValue visitUREM(SDNode *N); 199 SDValue visitMULHU(SDNode *N); 200 SDValue visitMULHS(SDNode *N); 201 SDValue visitSMUL_LOHI(SDNode *N); 202 SDValue visitUMUL_LOHI(SDNode *N); 203 SDValue visitSMULO(SDNode *N); 204 SDValue visitUMULO(SDNode *N); 205 SDValue visitSDIVREM(SDNode *N); 206 SDValue visitUDIVREM(SDNode *N); 207 SDValue visitAND(SDNode *N); 208 SDValue visitOR(SDNode *N); 209 SDValue visitXOR(SDNode *N); 210 SDValue SimplifyVBinOp(SDNode *N); 211 SDValue SimplifyVUnaryOp(SDNode *N); 212 SDValue visitSHL(SDNode *N); 213 SDValue visitSRA(SDNode *N); 214 SDValue visitSRL(SDNode *N); 215 SDValue visitCTLZ(SDNode *N); 216 SDValue visitCTLZ_ZERO_UNDEF(SDNode *N); 217 SDValue visitCTTZ(SDNode *N); 218 SDValue visitCTTZ_ZERO_UNDEF(SDNode *N); 219 SDValue visitCTPOP(SDNode *N); 220 SDValue visitSELECT(SDNode *N); 221 SDValue visitVSELECT(SDNode *N); 222 SDValue visitSELECT_CC(SDNode *N); 223 SDValue visitSETCC(SDNode *N); 224 SDValue visitSIGN_EXTEND(SDNode *N); 225 SDValue visitZERO_EXTEND(SDNode *N); 226 SDValue visitANY_EXTEND(SDNode *N); 227 SDValue visitSIGN_EXTEND_INREG(SDNode *N); 228 SDValue visitTRUNCATE(SDNode *N); 229 SDValue visitBITCAST(SDNode *N); 230 SDValue visitBUILD_PAIR(SDNode *N); 231 SDValue visitFADD(SDNode *N); 232 SDValue visitFSUB(SDNode *N); 233 SDValue visitFMUL(SDNode *N); 234 SDValue visitFMA(SDNode *N); 235 SDValue visitFDIV(SDNode *N); 236 SDValue visitFREM(SDNode *N); 237 SDValue visitFCOPYSIGN(SDNode *N); 238 SDValue visitSINT_TO_FP(SDNode *N); 239 SDValue visitUINT_TO_FP(SDNode *N); 240 SDValue visitFP_TO_SINT(SDNode *N); 241 SDValue visitFP_TO_UINT(SDNode *N); 242 SDValue visitFP_ROUND(SDNode *N); 243 SDValue visitFP_ROUND_INREG(SDNode *N); 244 SDValue visitFP_EXTEND(SDNode *N); 245 SDValue visitFNEG(SDNode *N); 246 SDValue visitFABS(SDNode *N); 247 SDValue visitFCEIL(SDNode *N); 248 SDValue visitFTRUNC(SDNode *N); 249 SDValue visitFFLOOR(SDNode *N); 250 SDValue visitBRCOND(SDNode *N); 251 SDValue visitBR_CC(SDNode *N); 252 SDValue visitLOAD(SDNode *N); 253 SDValue visitSTORE(SDNode *N); 254 SDValue visitINSERT_VECTOR_ELT(SDNode *N); 255 SDValue visitEXTRACT_VECTOR_ELT(SDNode *N); 256 SDValue visitBUILD_VECTOR(SDNode *N); 257 SDValue visitCONCAT_VECTORS(SDNode *N); 258 SDValue visitEXTRACT_SUBVECTOR(SDNode *N); 259 SDValue visitVECTOR_SHUFFLE(SDNode *N); 260 261 SDValue XformToShuffleWithZero(SDNode *N); 262 SDValue ReassociateOps(unsigned Opc, SDLoc DL, SDValue LHS, SDValue RHS); 263 264 SDValue visitShiftByConstant(SDNode *N, unsigned Amt); 265 266 bool SimplifySelectOps(SDNode *SELECT, SDValue LHS, SDValue RHS); 267 SDValue SimplifyBinOpWithSameOpcodeHands(SDNode *N); 268 SDValue SimplifySelect(SDLoc DL, SDValue N0, SDValue N1, SDValue N2); 269 SDValue SimplifySelectCC(SDLoc DL, SDValue N0, SDValue N1, SDValue N2, 270 SDValue N3, ISD::CondCode CC, 271 bool NotExtCompare = false); 272 SDValue SimplifySetCC(EVT VT, SDValue N0, SDValue N1, ISD::CondCode Cond, 273 SDLoc DL, bool foldBooleans = true); 274 SDValue SimplifyNodeWithTwoResults(SDNode *N, unsigned LoOp, 275 unsigned HiOp); 276 SDValue CombineConsecutiveLoads(SDNode *N, EVT VT); 277 SDValue ConstantFoldBITCASTofBUILD_VECTOR(SDNode *, EVT); 278 SDValue BuildSDIV(SDNode *N); 279 SDValue BuildUDIV(SDNode *N); 280 SDValue MatchBSwapHWordLow(SDNode *N, SDValue N0, SDValue N1, 281 bool DemandHighBits = true); 282 SDValue MatchBSwapHWord(SDNode *N, SDValue N0, SDValue N1); 283 SDNode *MatchRotate(SDValue LHS, SDValue RHS, SDLoc DL); 284 SDValue ReduceLoadWidth(SDNode *N); 285 SDValue ReduceLoadOpStoreWidth(SDNode *N); 286 SDValue TransformFPLoadStorePair(SDNode *N); 287 SDValue reduceBuildVecExtToExtBuildVec(SDNode *N); 288 SDValue reduceBuildVecConvertToConvertBuildVec(SDNode *N); 289 290 SDValue GetDemandedBits(SDValue V, const APInt &Mask); 291 292 /// GatherAllAliases - Walk up chain skipping non-aliasing memory nodes, 293 /// looking for aliasing nodes and adding them to the Aliases vector. 294 void GatherAllAliases(SDNode *N, SDValue OriginalChain, 295 SmallVectorImpl<SDValue> &Aliases); 296 297 /// isAlias - Return true if there is any possibility that the two addresses 298 /// overlap. 299 bool isAlias(SDValue Ptr1, int64_t Size1, bool IsVolatile1, 300 const Value *SrcValue1, int SrcValueOffset1, 301 unsigned SrcValueAlign1, 302 const MDNode *TBAAInfo1, 303 SDValue Ptr2, int64_t Size2, bool IsVolatile2, 304 const Value *SrcValue2, int SrcValueOffset2, 305 unsigned SrcValueAlign2, 306 const MDNode *TBAAInfo2) const; 307 308 /// isAlias - Return true if there is any possibility that the two addresses 309 /// overlap. 310 bool isAlias(LSBaseSDNode *Op0, LSBaseSDNode *Op1); 311 312 /// FindAliasInfo - Extracts the relevant alias information from the memory 313 /// node. Returns true if the operand was a load. 314 bool FindAliasInfo(SDNode *N, 315 SDValue &Ptr, int64_t &Size, bool &IsVolatile, 316 const Value *&SrcValue, int &SrcValueOffset, 317 unsigned &SrcValueAlignment, 318 const MDNode *&TBAAInfo) const; 319 320 /// FindBetterChain - Walk up chain skipping non-aliasing memory nodes, 321 /// looking for a better chain (aliasing node.) 322 SDValue FindBetterChain(SDNode *N, SDValue Chain); 323 324 /// Merge consecutive store operations into a wide store. 325 /// This optimization uses wide integers or vectors when possible. 326 /// \return True if some memory operations were changed. 327 bool MergeConsecutiveStores(StoreSDNode *N); 328 329 public: 330 DAGCombiner(SelectionDAG &D, AliasAnalysis &A, CodeGenOpt::Level OL) 331 : DAG(D), TLI(D.getTargetLoweringInfo()), Level(BeforeLegalizeTypes), 332 OptLevel(OL), LegalOperations(false), LegalTypes(false), AA(A) { 333 AttributeSet FnAttrs = 334 DAG.getMachineFunction().getFunction()->getAttributes(); 335 ForCodeSize = 336 FnAttrs.hasAttribute(AttributeSet::FunctionIndex, 337 Attribute::OptimizeForSize) || 338 FnAttrs.hasAttribute(AttributeSet::FunctionIndex, Attribute::MinSize); 339 } 340 341 /// Run - runs the dag combiner on all nodes in the work list 342 void Run(CombineLevel AtLevel); 343 344 SelectionDAG &getDAG() const { return DAG; } 345 346 /// getShiftAmountTy - Returns a type large enough to hold any valid 347 /// shift amount - before type legalization these can be huge. 348 EVT getShiftAmountTy(EVT LHSTy) { 349 assert(LHSTy.isInteger() && "Shift amount is not an integer type!"); 350 if (LHSTy.isVector()) 351 return LHSTy; 352 return LegalTypes ? TLI.getScalarShiftAmountTy(LHSTy) 353 : TLI.getPointerTy(); 354 } 355 356 /// isTypeLegal - This method returns true if we are running before type 357 /// legalization or if the specified VT is legal. 358 bool isTypeLegal(const EVT &VT) { 359 if (!LegalTypes) return true; 360 return TLI.isTypeLegal(VT); 361 } 362 363 /// getSetCCResultType - Convenience wrapper around 364 /// TargetLowering::getSetCCResultType 365 EVT getSetCCResultType(EVT VT) const { 366 return TLI.getSetCCResultType(*DAG.getContext(), VT); 367 } 368 }; 369} 370 371 372namespace { 373/// WorkListRemover - This class is a DAGUpdateListener that removes any deleted 374/// nodes from the worklist. 375class WorkListRemover : public SelectionDAG::DAGUpdateListener { 376 DAGCombiner &DC; 377public: 378 explicit WorkListRemover(DAGCombiner &dc) 379 : SelectionDAG::DAGUpdateListener(dc.getDAG()), DC(dc) {} 380 381 virtual void NodeDeleted(SDNode *N, SDNode *E) { 382 DC.removeFromWorkList(N); 383 } 384}; 385} 386 387//===----------------------------------------------------------------------===// 388// TargetLowering::DAGCombinerInfo implementation 389//===----------------------------------------------------------------------===// 390 391void TargetLowering::DAGCombinerInfo::AddToWorklist(SDNode *N) { 392 ((DAGCombiner*)DC)->AddToWorkList(N); 393} 394 395void TargetLowering::DAGCombinerInfo::RemoveFromWorklist(SDNode *N) { 396 ((DAGCombiner*)DC)->removeFromWorkList(N); 397} 398 399SDValue TargetLowering::DAGCombinerInfo:: 400CombineTo(SDNode *N, const std::vector<SDValue> &To, bool AddTo) { 401 return ((DAGCombiner*)DC)->CombineTo(N, &To[0], To.size(), AddTo); 402} 403 404SDValue TargetLowering::DAGCombinerInfo:: 405CombineTo(SDNode *N, SDValue Res, bool AddTo) { 406 return ((DAGCombiner*)DC)->CombineTo(N, Res, AddTo); 407} 408 409 410SDValue TargetLowering::DAGCombinerInfo:: 411CombineTo(SDNode *N, SDValue Res0, SDValue Res1, bool AddTo) { 412 return ((DAGCombiner*)DC)->CombineTo(N, Res0, Res1, AddTo); 413} 414 415void TargetLowering::DAGCombinerInfo:: 416CommitTargetLoweringOpt(const TargetLowering::TargetLoweringOpt &TLO) { 417 return ((DAGCombiner*)DC)->CommitTargetLoweringOpt(TLO); 418} 419 420//===----------------------------------------------------------------------===// 421// Helper Functions 422//===----------------------------------------------------------------------===// 423 424/// isNegatibleForFree - Return 1 if we can compute the negated form of the 425/// specified expression for the same cost as the expression itself, or 2 if we 426/// can compute the negated form more cheaply than the expression itself. 427static char isNegatibleForFree(SDValue Op, bool LegalOperations, 428 const TargetLowering &TLI, 429 const TargetOptions *Options, 430 unsigned Depth = 0) { 431 // fneg is removable even if it has multiple uses. 432 if (Op.getOpcode() == ISD::FNEG) return 2; 433 434 // Don't allow anything with multiple uses. 435 if (!Op.hasOneUse()) return 0; 436 437 // Don't recurse exponentially. 438 if (Depth > 6) return 0; 439 440 switch (Op.getOpcode()) { 441 default: return false; 442 case ISD::ConstantFP: 443 // Don't invert constant FP values after legalize. The negated constant 444 // isn't necessarily legal. 445 return LegalOperations ? 0 : 1; 446 case ISD::FADD: 447 // FIXME: determine better conditions for this xform. 448 if (!Options->UnsafeFPMath) return 0; 449 450 // After operation legalization, it might not be legal to create new FSUBs. 451 if (LegalOperations && 452 !TLI.isOperationLegalOrCustom(ISD::FSUB, Op.getValueType())) 453 return 0; 454 455 // fold (fneg (fadd A, B)) -> (fsub (fneg A), B) 456 if (char V = isNegatibleForFree(Op.getOperand(0), LegalOperations, TLI, 457 Options, Depth + 1)) 458 return V; 459 // fold (fneg (fadd A, B)) -> (fsub (fneg B), A) 460 return isNegatibleForFree(Op.getOperand(1), LegalOperations, TLI, Options, 461 Depth + 1); 462 case ISD::FSUB: 463 // We can't turn -(A-B) into B-A when we honor signed zeros. 464 if (!Options->UnsafeFPMath) return 0; 465 466 // fold (fneg (fsub A, B)) -> (fsub B, A) 467 return 1; 468 469 case ISD::FMUL: 470 case ISD::FDIV: 471 if (Options->HonorSignDependentRoundingFPMath()) return 0; 472 473 // fold (fneg (fmul X, Y)) -> (fmul (fneg X), Y) or (fmul X, (fneg Y)) 474 if (char V = isNegatibleForFree(Op.getOperand(0), LegalOperations, TLI, 475 Options, Depth + 1)) 476 return V; 477 478 return isNegatibleForFree(Op.getOperand(1), LegalOperations, TLI, Options, 479 Depth + 1); 480 481 case ISD::FP_EXTEND: 482 case ISD::FP_ROUND: 483 case ISD::FSIN: 484 return isNegatibleForFree(Op.getOperand(0), LegalOperations, TLI, Options, 485 Depth + 1); 486 } 487} 488 489/// GetNegatedExpression - If isNegatibleForFree returns true, this function 490/// returns the newly negated expression. 491static SDValue GetNegatedExpression(SDValue Op, SelectionDAG &DAG, 492 bool LegalOperations, unsigned Depth = 0) { 493 // fneg is removable even if it has multiple uses. 494 if (Op.getOpcode() == ISD::FNEG) return Op.getOperand(0); 495 496 // Don't allow anything with multiple uses. 497 assert(Op.hasOneUse() && "Unknown reuse!"); 498 499 assert(Depth <= 6 && "GetNegatedExpression doesn't match isNegatibleForFree"); 500 switch (Op.getOpcode()) { 501 default: llvm_unreachable("Unknown code"); 502 case ISD::ConstantFP: { 503 APFloat V = cast<ConstantFPSDNode>(Op)->getValueAPF(); 504 V.changeSign(); 505 return DAG.getConstantFP(V, Op.getValueType()); 506 } 507 case ISD::FADD: 508 // FIXME: determine better conditions for this xform. 509 assert(DAG.getTarget().Options.UnsafeFPMath); 510 511 // fold (fneg (fadd A, B)) -> (fsub (fneg A), B) 512 if (isNegatibleForFree(Op.getOperand(0), LegalOperations, 513 DAG.getTargetLoweringInfo(), 514 &DAG.getTarget().Options, Depth+1)) 515 return DAG.getNode(ISD::FSUB, SDLoc(Op), Op.getValueType(), 516 GetNegatedExpression(Op.getOperand(0), DAG, 517 LegalOperations, Depth+1), 518 Op.getOperand(1)); 519 // fold (fneg (fadd A, B)) -> (fsub (fneg B), A) 520 return DAG.getNode(ISD::FSUB, SDLoc(Op), Op.getValueType(), 521 GetNegatedExpression(Op.getOperand(1), DAG, 522 LegalOperations, Depth+1), 523 Op.getOperand(0)); 524 case ISD::FSUB: 525 // We can't turn -(A-B) into B-A when we honor signed zeros. 526 assert(DAG.getTarget().Options.UnsafeFPMath); 527 528 // fold (fneg (fsub 0, B)) -> B 529 if (ConstantFPSDNode *N0CFP = dyn_cast<ConstantFPSDNode>(Op.getOperand(0))) 530 if (N0CFP->getValueAPF().isZero()) 531 return Op.getOperand(1); 532 533 // fold (fneg (fsub A, B)) -> (fsub B, A) 534 return DAG.getNode(ISD::FSUB, SDLoc(Op), Op.getValueType(), 535 Op.getOperand(1), Op.getOperand(0)); 536 537 case ISD::FMUL: 538 case ISD::FDIV: 539 assert(!DAG.getTarget().Options.HonorSignDependentRoundingFPMath()); 540 541 // fold (fneg (fmul X, Y)) -> (fmul (fneg X), Y) 542 if (isNegatibleForFree(Op.getOperand(0), LegalOperations, 543 DAG.getTargetLoweringInfo(), 544 &DAG.getTarget().Options, Depth+1)) 545 return DAG.getNode(Op.getOpcode(), SDLoc(Op), Op.getValueType(), 546 GetNegatedExpression(Op.getOperand(0), DAG, 547 LegalOperations, Depth+1), 548 Op.getOperand(1)); 549 550 // fold (fneg (fmul X, Y)) -> (fmul X, (fneg Y)) 551 return DAG.getNode(Op.getOpcode(), SDLoc(Op), Op.getValueType(), 552 Op.getOperand(0), 553 GetNegatedExpression(Op.getOperand(1), DAG, 554 LegalOperations, Depth+1)); 555 556 case ISD::FP_EXTEND: 557 case ISD::FSIN: 558 return DAG.getNode(Op.getOpcode(), SDLoc(Op), Op.getValueType(), 559 GetNegatedExpression(Op.getOperand(0), DAG, 560 LegalOperations, Depth+1)); 561 case ISD::FP_ROUND: 562 return DAG.getNode(ISD::FP_ROUND, SDLoc(Op), Op.getValueType(), 563 GetNegatedExpression(Op.getOperand(0), DAG, 564 LegalOperations, Depth+1), 565 Op.getOperand(1)); 566 } 567} 568 569 570// isSetCCEquivalent - Return true if this node is a setcc, or is a select_cc 571// that selects between the values 1 and 0, making it equivalent to a setcc. 572// Also, set the incoming LHS, RHS, and CC references to the appropriate 573// nodes based on the type of node we are checking. This simplifies life a 574// bit for the callers. 575static bool isSetCCEquivalent(SDValue N, SDValue &LHS, SDValue &RHS, 576 SDValue &CC) { 577 if (N.getOpcode() == ISD::SETCC) { 578 LHS = N.getOperand(0); 579 RHS = N.getOperand(1); 580 CC = N.getOperand(2); 581 return true; 582 } 583 if (N.getOpcode() == ISD::SELECT_CC && 584 N.getOperand(2).getOpcode() == ISD::Constant && 585 N.getOperand(3).getOpcode() == ISD::Constant && 586 cast<ConstantSDNode>(N.getOperand(2))->getAPIntValue() == 1 && 587 cast<ConstantSDNode>(N.getOperand(3))->isNullValue()) { 588 LHS = N.getOperand(0); 589 RHS = N.getOperand(1); 590 CC = N.getOperand(4); 591 return true; 592 } 593 return false; 594} 595 596// isOneUseSetCC - Return true if this is a SetCC-equivalent operation with only 597// one use. If this is true, it allows the users to invert the operation for 598// free when it is profitable to do so. 599static bool isOneUseSetCC(SDValue N) { 600 SDValue N0, N1, N2; 601 if (isSetCCEquivalent(N, N0, N1, N2) && N.getNode()->hasOneUse()) 602 return true; 603 return false; 604} 605 606SDValue DAGCombiner::ReassociateOps(unsigned Opc, SDLoc DL, 607 SDValue N0, SDValue N1) { 608 EVT VT = N0.getValueType(); 609 if (N0.getOpcode() == Opc && isa<ConstantSDNode>(N0.getOperand(1))) { 610 if (isa<ConstantSDNode>(N1)) { 611 // reassoc. (op (op x, c1), c2) -> (op x, (op c1, c2)) 612 SDValue OpNode = 613 DAG.FoldConstantArithmetic(Opc, VT, 614 cast<ConstantSDNode>(N0.getOperand(1)), 615 cast<ConstantSDNode>(N1)); 616 return DAG.getNode(Opc, DL, VT, N0.getOperand(0), OpNode); 617 } 618 if (N0.hasOneUse()) { 619 // reassoc. (op (op x, c1), y) -> (op (op x, y), c1) iff x+c1 has one use 620 SDValue OpNode = DAG.getNode(Opc, SDLoc(N0), VT, 621 N0.getOperand(0), N1); 622 AddToWorkList(OpNode.getNode()); 623 return DAG.getNode(Opc, DL, VT, OpNode, N0.getOperand(1)); 624 } 625 } 626 627 if (N1.getOpcode() == Opc && isa<ConstantSDNode>(N1.getOperand(1))) { 628 if (isa<ConstantSDNode>(N0)) { 629 // reassoc. (op c2, (op x, c1)) -> (op x, (op c1, c2)) 630 SDValue OpNode = 631 DAG.FoldConstantArithmetic(Opc, VT, 632 cast<ConstantSDNode>(N1.getOperand(1)), 633 cast<ConstantSDNode>(N0)); 634 return DAG.getNode(Opc, DL, VT, N1.getOperand(0), OpNode); 635 } 636 if (N1.hasOneUse()) { 637 // reassoc. (op y, (op x, c1)) -> (op (op x, y), c1) iff x+c1 has one use 638 SDValue OpNode = DAG.getNode(Opc, SDLoc(N0), VT, 639 N1.getOperand(0), N0); 640 AddToWorkList(OpNode.getNode()); 641 return DAG.getNode(Opc, DL, VT, OpNode, N1.getOperand(1)); 642 } 643 } 644 645 return SDValue(); 646} 647 648SDValue DAGCombiner::CombineTo(SDNode *N, const SDValue *To, unsigned NumTo, 649 bool AddTo) { 650 assert(N->getNumValues() == NumTo && "Broken CombineTo call!"); 651 ++NodesCombined; 652 DEBUG(dbgs() << "\nReplacing.1 "; 653 N->dump(&DAG); 654 dbgs() << "\nWith: "; 655 To[0].getNode()->dump(&DAG); 656 dbgs() << " and " << NumTo-1 << " other values\n"; 657 for (unsigned i = 0, e = NumTo; i != e; ++i) 658 assert((!To[i].getNode() || 659 N->getValueType(i) == To[i].getValueType()) && 660 "Cannot combine value to value of different type!")); 661 WorkListRemover DeadNodes(*this); 662 DAG.ReplaceAllUsesWith(N, To); 663 if (AddTo) { 664 // Push the new nodes and any users onto the worklist 665 for (unsigned i = 0, e = NumTo; i != e; ++i) { 666 if (To[i].getNode()) { 667 AddToWorkList(To[i].getNode()); 668 AddUsersToWorkList(To[i].getNode()); 669 } 670 } 671 } 672 673 // Finally, if the node is now dead, remove it from the graph. The node 674 // may not be dead if the replacement process recursively simplified to 675 // something else needing this node. 676 if (N->use_empty()) { 677 // Nodes can be reintroduced into the worklist. Make sure we do not 678 // process a node that has been replaced. 679 removeFromWorkList(N); 680 681 // Finally, since the node is now dead, remove it from the graph. 682 DAG.DeleteNode(N); 683 } 684 return SDValue(N, 0); 685} 686 687void DAGCombiner:: 688CommitTargetLoweringOpt(const TargetLowering::TargetLoweringOpt &TLO) { 689 // Replace all uses. If any nodes become isomorphic to other nodes and 690 // are deleted, make sure to remove them from our worklist. 691 WorkListRemover DeadNodes(*this); 692 DAG.ReplaceAllUsesOfValueWith(TLO.Old, TLO.New); 693 694 // Push the new node and any (possibly new) users onto the worklist. 695 AddToWorkList(TLO.New.getNode()); 696 AddUsersToWorkList(TLO.New.getNode()); 697 698 // Finally, if the node is now dead, remove it from the graph. The node 699 // may not be dead if the replacement process recursively simplified to 700 // something else needing this node. 701 if (TLO.Old.getNode()->use_empty()) { 702 removeFromWorkList(TLO.Old.getNode()); 703 704 // If the operands of this node are only used by the node, they will now 705 // be dead. Make sure to visit them first to delete dead nodes early. 706 for (unsigned i = 0, e = TLO.Old.getNode()->getNumOperands(); i != e; ++i) 707 if (TLO.Old.getNode()->getOperand(i).getNode()->hasOneUse()) 708 AddToWorkList(TLO.Old.getNode()->getOperand(i).getNode()); 709 710 DAG.DeleteNode(TLO.Old.getNode()); 711 } 712} 713 714/// SimplifyDemandedBits - Check the specified integer node value to see if 715/// it can be simplified or if things it uses can be simplified by bit 716/// propagation. If so, return true. 717bool DAGCombiner::SimplifyDemandedBits(SDValue Op, const APInt &Demanded) { 718 TargetLowering::TargetLoweringOpt TLO(DAG, LegalTypes, LegalOperations); 719 APInt KnownZero, KnownOne; 720 if (!TLI.SimplifyDemandedBits(Op, Demanded, KnownZero, KnownOne, TLO)) 721 return false; 722 723 // Revisit the node. 724 AddToWorkList(Op.getNode()); 725 726 // Replace the old value with the new one. 727 ++NodesCombined; 728 DEBUG(dbgs() << "\nReplacing.2 "; 729 TLO.Old.getNode()->dump(&DAG); 730 dbgs() << "\nWith: "; 731 TLO.New.getNode()->dump(&DAG); 732 dbgs() << '\n'); 733 734 CommitTargetLoweringOpt(TLO); 735 return true; 736} 737 738void DAGCombiner::ReplaceLoadWithPromotedLoad(SDNode *Load, SDNode *ExtLoad) { 739 SDLoc dl(Load); 740 EVT VT = Load->getValueType(0); 741 SDValue Trunc = DAG.getNode(ISD::TRUNCATE, dl, VT, SDValue(ExtLoad, 0)); 742 743 DEBUG(dbgs() << "\nReplacing.9 "; 744 Load->dump(&DAG); 745 dbgs() << "\nWith: "; 746 Trunc.getNode()->dump(&DAG); 747 dbgs() << '\n'); 748 WorkListRemover DeadNodes(*this); 749 DAG.ReplaceAllUsesOfValueWith(SDValue(Load, 0), Trunc); 750 DAG.ReplaceAllUsesOfValueWith(SDValue(Load, 1), SDValue(ExtLoad, 1)); 751 removeFromWorkList(Load); 752 DAG.DeleteNode(Load); 753 AddToWorkList(Trunc.getNode()); 754} 755 756SDValue DAGCombiner::PromoteOperand(SDValue Op, EVT PVT, bool &Replace) { 757 Replace = false; 758 SDLoc dl(Op); 759 if (LoadSDNode *LD = dyn_cast<LoadSDNode>(Op)) { 760 EVT MemVT = LD->getMemoryVT(); 761 ISD::LoadExtType ExtType = ISD::isNON_EXTLoad(LD) 762 ? (TLI.isLoadExtLegal(ISD::ZEXTLOAD, MemVT) ? ISD::ZEXTLOAD 763 : ISD::EXTLOAD) 764 : LD->getExtensionType(); 765 Replace = true; 766 return DAG.getExtLoad(ExtType, dl, PVT, 767 LD->getChain(), LD->getBasePtr(), 768 MemVT, LD->getMemOperand()); 769 } 770 771 unsigned Opc = Op.getOpcode(); 772 switch (Opc) { 773 default: break; 774 case ISD::AssertSext: 775 return DAG.getNode(ISD::AssertSext, dl, PVT, 776 SExtPromoteOperand(Op.getOperand(0), PVT), 777 Op.getOperand(1)); 778 case ISD::AssertZext: 779 return DAG.getNode(ISD::AssertZext, dl, PVT, 780 ZExtPromoteOperand(Op.getOperand(0), PVT), 781 Op.getOperand(1)); 782 case ISD::Constant: { 783 unsigned ExtOpc = 784 Op.getValueType().isByteSized() ? ISD::SIGN_EXTEND : ISD::ZERO_EXTEND; 785 return DAG.getNode(ExtOpc, dl, PVT, Op); 786 } 787 } 788 789 if (!TLI.isOperationLegal(ISD::ANY_EXTEND, PVT)) 790 return SDValue(); 791 return DAG.getNode(ISD::ANY_EXTEND, dl, PVT, Op); 792} 793 794SDValue DAGCombiner::SExtPromoteOperand(SDValue Op, EVT PVT) { 795 if (!TLI.isOperationLegal(ISD::SIGN_EXTEND_INREG, PVT)) 796 return SDValue(); 797 EVT OldVT = Op.getValueType(); 798 SDLoc dl(Op); 799 bool Replace = false; 800 SDValue NewOp = PromoteOperand(Op, PVT, Replace); 801 if (NewOp.getNode() == 0) 802 return SDValue(); 803 AddToWorkList(NewOp.getNode()); 804 805 if (Replace) 806 ReplaceLoadWithPromotedLoad(Op.getNode(), NewOp.getNode()); 807 return DAG.getNode(ISD::SIGN_EXTEND_INREG, dl, NewOp.getValueType(), NewOp, 808 DAG.getValueType(OldVT)); 809} 810 811SDValue DAGCombiner::ZExtPromoteOperand(SDValue Op, EVT PVT) { 812 EVT OldVT = Op.getValueType(); 813 SDLoc dl(Op); 814 bool Replace = false; 815 SDValue NewOp = PromoteOperand(Op, PVT, Replace); 816 if (NewOp.getNode() == 0) 817 return SDValue(); 818 AddToWorkList(NewOp.getNode()); 819 820 if (Replace) 821 ReplaceLoadWithPromotedLoad(Op.getNode(), NewOp.getNode()); 822 return DAG.getZeroExtendInReg(NewOp, dl, OldVT); 823} 824 825/// PromoteIntBinOp - Promote the specified integer binary operation if the 826/// target indicates it is beneficial. e.g. On x86, it's usually better to 827/// promote i16 operations to i32 since i16 instructions are longer. 828SDValue DAGCombiner::PromoteIntBinOp(SDValue Op) { 829 if (!LegalOperations) 830 return SDValue(); 831 832 EVT VT = Op.getValueType(); 833 if (VT.isVector() || !VT.isInteger()) 834 return SDValue(); 835 836 // If operation type is 'undesirable', e.g. i16 on x86, consider 837 // promoting it. 838 unsigned Opc = Op.getOpcode(); 839 if (TLI.isTypeDesirableForOp(Opc, VT)) 840 return SDValue(); 841 842 EVT PVT = VT; 843 // Consult target whether it is a good idea to promote this operation and 844 // what's the right type to promote it to. 845 if (TLI.IsDesirableToPromoteOp(Op, PVT)) { 846 assert(PVT != VT && "Don't know what type to promote to!"); 847 848 bool Replace0 = false; 849 SDValue N0 = Op.getOperand(0); 850 SDValue NN0 = PromoteOperand(N0, PVT, Replace0); 851 if (NN0.getNode() == 0) 852 return SDValue(); 853 854 bool Replace1 = false; 855 SDValue N1 = Op.getOperand(1); 856 SDValue NN1; 857 if (N0 == N1) 858 NN1 = NN0; 859 else { 860 NN1 = PromoteOperand(N1, PVT, Replace1); 861 if (NN1.getNode() == 0) 862 return SDValue(); 863 } 864 865 AddToWorkList(NN0.getNode()); 866 if (NN1.getNode()) 867 AddToWorkList(NN1.getNode()); 868 869 if (Replace0) 870 ReplaceLoadWithPromotedLoad(N0.getNode(), NN0.getNode()); 871 if (Replace1) 872 ReplaceLoadWithPromotedLoad(N1.getNode(), NN1.getNode()); 873 874 DEBUG(dbgs() << "\nPromoting "; 875 Op.getNode()->dump(&DAG)); 876 SDLoc dl(Op); 877 return DAG.getNode(ISD::TRUNCATE, dl, VT, 878 DAG.getNode(Opc, dl, PVT, NN0, NN1)); 879 } 880 return SDValue(); 881} 882 883/// PromoteIntShiftOp - Promote the specified integer shift operation if the 884/// target indicates it is beneficial. e.g. On x86, it's usually better to 885/// promote i16 operations to i32 since i16 instructions are longer. 886SDValue DAGCombiner::PromoteIntShiftOp(SDValue Op) { 887 if (!LegalOperations) 888 return SDValue(); 889 890 EVT VT = Op.getValueType(); 891 if (VT.isVector() || !VT.isInteger()) 892 return SDValue(); 893 894 // If operation type is 'undesirable', e.g. i16 on x86, consider 895 // promoting it. 896 unsigned Opc = Op.getOpcode(); 897 if (TLI.isTypeDesirableForOp(Opc, VT)) 898 return SDValue(); 899 900 EVT PVT = VT; 901 // Consult target whether it is a good idea to promote this operation and 902 // what's the right type to promote it to. 903 if (TLI.IsDesirableToPromoteOp(Op, PVT)) { 904 assert(PVT != VT && "Don't know what type to promote to!"); 905 906 bool Replace = false; 907 SDValue N0 = Op.getOperand(0); 908 if (Opc == ISD::SRA) 909 N0 = SExtPromoteOperand(Op.getOperand(0), PVT); 910 else if (Opc == ISD::SRL) 911 N0 = ZExtPromoteOperand(Op.getOperand(0), PVT); 912 else 913 N0 = PromoteOperand(N0, PVT, Replace); 914 if (N0.getNode() == 0) 915 return SDValue(); 916 917 AddToWorkList(N0.getNode()); 918 if (Replace) 919 ReplaceLoadWithPromotedLoad(Op.getOperand(0).getNode(), N0.getNode()); 920 921 DEBUG(dbgs() << "\nPromoting "; 922 Op.getNode()->dump(&DAG)); 923 SDLoc dl(Op); 924 return DAG.getNode(ISD::TRUNCATE, dl, VT, 925 DAG.getNode(Opc, dl, PVT, N0, Op.getOperand(1))); 926 } 927 return SDValue(); 928} 929 930SDValue DAGCombiner::PromoteExtend(SDValue Op) { 931 if (!LegalOperations) 932 return SDValue(); 933 934 EVT VT = Op.getValueType(); 935 if (VT.isVector() || !VT.isInteger()) 936 return SDValue(); 937 938 // If operation type is 'undesirable', e.g. i16 on x86, consider 939 // promoting it. 940 unsigned Opc = Op.getOpcode(); 941 if (TLI.isTypeDesirableForOp(Opc, VT)) 942 return SDValue(); 943 944 EVT PVT = VT; 945 // Consult target whether it is a good idea to promote this operation and 946 // what's the right type to promote it to. 947 if (TLI.IsDesirableToPromoteOp(Op, PVT)) { 948 assert(PVT != VT && "Don't know what type to promote to!"); 949 // fold (aext (aext x)) -> (aext x) 950 // fold (aext (zext x)) -> (zext x) 951 // fold (aext (sext x)) -> (sext x) 952 DEBUG(dbgs() << "\nPromoting "; 953 Op.getNode()->dump(&DAG)); 954 return DAG.getNode(Op.getOpcode(), SDLoc(Op), VT, Op.getOperand(0)); 955 } 956 return SDValue(); 957} 958 959bool DAGCombiner::PromoteLoad(SDValue Op) { 960 if (!LegalOperations) 961 return false; 962 963 EVT VT = Op.getValueType(); 964 if (VT.isVector() || !VT.isInteger()) 965 return false; 966 967 // If operation type is 'undesirable', e.g. i16 on x86, consider 968 // promoting it. 969 unsigned Opc = Op.getOpcode(); 970 if (TLI.isTypeDesirableForOp(Opc, VT)) 971 return false; 972 973 EVT PVT = VT; 974 // Consult target whether it is a good idea to promote this operation and 975 // what's the right type to promote it to. 976 if (TLI.IsDesirableToPromoteOp(Op, PVT)) { 977 assert(PVT != VT && "Don't know what type to promote to!"); 978 979 SDLoc dl(Op); 980 SDNode *N = Op.getNode(); 981 LoadSDNode *LD = cast<LoadSDNode>(N); 982 EVT MemVT = LD->getMemoryVT(); 983 ISD::LoadExtType ExtType = ISD::isNON_EXTLoad(LD) 984 ? (TLI.isLoadExtLegal(ISD::ZEXTLOAD, MemVT) ? ISD::ZEXTLOAD 985 : ISD::EXTLOAD) 986 : LD->getExtensionType(); 987 SDValue NewLD = DAG.getExtLoad(ExtType, dl, PVT, 988 LD->getChain(), LD->getBasePtr(), 989 MemVT, LD->getMemOperand()); 990 SDValue Result = DAG.getNode(ISD::TRUNCATE, dl, VT, NewLD); 991 992 DEBUG(dbgs() << "\nPromoting "; 993 N->dump(&DAG); 994 dbgs() << "\nTo: "; 995 Result.getNode()->dump(&DAG); 996 dbgs() << '\n'); 997 WorkListRemover DeadNodes(*this); 998 DAG.ReplaceAllUsesOfValueWith(SDValue(N, 0), Result); 999 DAG.ReplaceAllUsesOfValueWith(SDValue(N, 1), NewLD.getValue(1)); 1000 removeFromWorkList(N); 1001 DAG.DeleteNode(N); 1002 AddToWorkList(Result.getNode()); 1003 return true; 1004 } 1005 return false; 1006} 1007 1008 1009//===----------------------------------------------------------------------===// 1010// Main DAG Combiner implementation 1011//===----------------------------------------------------------------------===// 1012 1013void DAGCombiner::Run(CombineLevel AtLevel) { 1014 // set the instance variables, so that the various visit routines may use it. 1015 Level = AtLevel; 1016 LegalOperations = Level >= AfterLegalizeVectorOps; 1017 LegalTypes = Level >= AfterLegalizeTypes; 1018 1019 // Add all the dag nodes to the worklist. 1020 for (SelectionDAG::allnodes_iterator I = DAG.allnodes_begin(), 1021 E = DAG.allnodes_end(); I != E; ++I) 1022 AddToWorkList(I); 1023 1024 // Create a dummy node (which is not added to allnodes), that adds a reference 1025 // to the root node, preventing it from being deleted, and tracking any 1026 // changes of the root. 1027 HandleSDNode Dummy(DAG.getRoot()); 1028 1029 // The root of the dag may dangle to deleted nodes until the dag combiner is 1030 // done. Set it to null to avoid confusion. 1031 DAG.setRoot(SDValue()); 1032 1033 // while the worklist isn't empty, find a node and 1034 // try and combine it. 1035 while (!WorkListContents.empty()) { 1036 SDNode *N; 1037 // The WorkListOrder holds the SDNodes in order, but it may contain 1038 // duplicates. 1039 // In order to avoid a linear scan, we use a set (O(log N)) to hold what the 1040 // worklist *should* contain, and check the node we want to visit is should 1041 // actually be visited. 1042 do { 1043 N = WorkListOrder.pop_back_val(); 1044 } while (!WorkListContents.erase(N)); 1045 1046 // If N has no uses, it is dead. Make sure to revisit all N's operands once 1047 // N is deleted from the DAG, since they too may now be dead or may have a 1048 // reduced number of uses, allowing other xforms. 1049 if (N->use_empty() && N != &Dummy) { 1050 for (unsigned i = 0, e = N->getNumOperands(); i != e; ++i) 1051 AddToWorkList(N->getOperand(i).getNode()); 1052 1053 DAG.DeleteNode(N); 1054 continue; 1055 } 1056 1057 SDValue RV = combine(N); 1058 1059 if (RV.getNode() == 0) 1060 continue; 1061 1062 ++NodesCombined; 1063 1064 // If we get back the same node we passed in, rather than a new node or 1065 // zero, we know that the node must have defined multiple values and 1066 // CombineTo was used. Since CombineTo takes care of the worklist 1067 // mechanics for us, we have no work to do in this case. 1068 if (RV.getNode() == N) 1069 continue; 1070 1071 assert(N->getOpcode() != ISD::DELETED_NODE && 1072 RV.getNode()->getOpcode() != ISD::DELETED_NODE && 1073 "Node was deleted but visit returned new node!"); 1074 1075 DEBUG(dbgs() << "\nReplacing.3 "; 1076 N->dump(&DAG); 1077 dbgs() << "\nWith: "; 1078 RV.getNode()->dump(&DAG); 1079 dbgs() << '\n'); 1080 1081 // Transfer debug value. 1082 DAG.TransferDbgValues(SDValue(N, 0), RV); 1083 WorkListRemover DeadNodes(*this); 1084 if (N->getNumValues() == RV.getNode()->getNumValues()) 1085 DAG.ReplaceAllUsesWith(N, RV.getNode()); 1086 else { 1087 assert(N->getValueType(0) == RV.getValueType() && 1088 N->getNumValues() == 1 && "Type mismatch"); 1089 SDValue OpV = RV; 1090 DAG.ReplaceAllUsesWith(N, &OpV); 1091 } 1092 1093 // Push the new node and any users onto the worklist 1094 AddToWorkList(RV.getNode()); 1095 AddUsersToWorkList(RV.getNode()); 1096 1097 // Add any uses of the old node to the worklist in case this node is the 1098 // last one that uses them. They may become dead after this node is 1099 // deleted. 1100 for (unsigned i = 0, e = N->getNumOperands(); i != e; ++i) 1101 AddToWorkList(N->getOperand(i).getNode()); 1102 1103 // Finally, if the node is now dead, remove it from the graph. The node 1104 // may not be dead if the replacement process recursively simplified to 1105 // something else needing this node. 1106 if (N->use_empty()) { 1107 // Nodes can be reintroduced into the worklist. Make sure we do not 1108 // process a node that has been replaced. 1109 removeFromWorkList(N); 1110 1111 // Finally, since the node is now dead, remove it from the graph. 1112 DAG.DeleteNode(N); 1113 } 1114 } 1115 1116 // If the root changed (e.g. it was a dead load, update the root). 1117 DAG.setRoot(Dummy.getValue()); 1118 DAG.RemoveDeadNodes(); 1119} 1120 1121SDValue DAGCombiner::visit(SDNode *N) { 1122 switch (N->getOpcode()) { 1123 default: break; 1124 case ISD::TokenFactor: return visitTokenFactor(N); 1125 case ISD::MERGE_VALUES: return visitMERGE_VALUES(N); 1126 case ISD::ADD: return visitADD(N); 1127 case ISD::SUB: return visitSUB(N); 1128 case ISD::ADDC: return visitADDC(N); 1129 case ISD::SUBC: return visitSUBC(N); 1130 case ISD::ADDE: return visitADDE(N); 1131 case ISD::SUBE: return visitSUBE(N); 1132 case ISD::MUL: return visitMUL(N); 1133 case ISD::SDIV: return visitSDIV(N); 1134 case ISD::UDIV: return visitUDIV(N); 1135 case ISD::SREM: return visitSREM(N); 1136 case ISD::UREM: return visitUREM(N); 1137 case ISD::MULHU: return visitMULHU(N); 1138 case ISD::MULHS: return visitMULHS(N); 1139 case ISD::SMUL_LOHI: return visitSMUL_LOHI(N); 1140 case ISD::UMUL_LOHI: return visitUMUL_LOHI(N); 1141 case ISD::SMULO: return visitSMULO(N); 1142 case ISD::UMULO: return visitUMULO(N); 1143 case ISD::SDIVREM: return visitSDIVREM(N); 1144 case ISD::UDIVREM: return visitUDIVREM(N); 1145 case ISD::AND: return visitAND(N); 1146 case ISD::OR: return visitOR(N); 1147 case ISD::XOR: return visitXOR(N); 1148 case ISD::SHL: return visitSHL(N); 1149 case ISD::SRA: return visitSRA(N); 1150 case ISD::SRL: return visitSRL(N); 1151 case ISD::CTLZ: return visitCTLZ(N); 1152 case ISD::CTLZ_ZERO_UNDEF: return visitCTLZ_ZERO_UNDEF(N); 1153 case ISD::CTTZ: return visitCTTZ(N); 1154 case ISD::CTTZ_ZERO_UNDEF: return visitCTTZ_ZERO_UNDEF(N); 1155 case ISD::CTPOP: return visitCTPOP(N); 1156 case ISD::SELECT: return visitSELECT(N); 1157 case ISD::VSELECT: return visitVSELECT(N); 1158 case ISD::SELECT_CC: return visitSELECT_CC(N); 1159 case ISD::SETCC: return visitSETCC(N); 1160 case ISD::SIGN_EXTEND: return visitSIGN_EXTEND(N); 1161 case ISD::ZERO_EXTEND: return visitZERO_EXTEND(N); 1162 case ISD::ANY_EXTEND: return visitANY_EXTEND(N); 1163 case ISD::SIGN_EXTEND_INREG: return visitSIGN_EXTEND_INREG(N); 1164 case ISD::TRUNCATE: return visitTRUNCATE(N); 1165 case ISD::BITCAST: return visitBITCAST(N); 1166 case ISD::BUILD_PAIR: return visitBUILD_PAIR(N); 1167 case ISD::FADD: return visitFADD(N); 1168 case ISD::FSUB: return visitFSUB(N); 1169 case ISD::FMUL: return visitFMUL(N); 1170 case ISD::FMA: return visitFMA(N); 1171 case ISD::FDIV: return visitFDIV(N); 1172 case ISD::FREM: return visitFREM(N); 1173 case ISD::FCOPYSIGN: return visitFCOPYSIGN(N); 1174 case ISD::SINT_TO_FP: return visitSINT_TO_FP(N); 1175 case ISD::UINT_TO_FP: return visitUINT_TO_FP(N); 1176 case ISD::FP_TO_SINT: return visitFP_TO_SINT(N); 1177 case ISD::FP_TO_UINT: return visitFP_TO_UINT(N); 1178 case ISD::FP_ROUND: return visitFP_ROUND(N); 1179 case ISD::FP_ROUND_INREG: return visitFP_ROUND_INREG(N); 1180 case ISD::FP_EXTEND: return visitFP_EXTEND(N); 1181 case ISD::FNEG: return visitFNEG(N); 1182 case ISD::FABS: return visitFABS(N); 1183 case ISD::FFLOOR: return visitFFLOOR(N); 1184 case ISD::FCEIL: return visitFCEIL(N); 1185 case ISD::FTRUNC: return visitFTRUNC(N); 1186 case ISD::BRCOND: return visitBRCOND(N); 1187 case ISD::BR_CC: return visitBR_CC(N); 1188 case ISD::LOAD: return visitLOAD(N); 1189 case ISD::STORE: return visitSTORE(N); 1190 case ISD::INSERT_VECTOR_ELT: return visitINSERT_VECTOR_ELT(N); 1191 case ISD::EXTRACT_VECTOR_ELT: return visitEXTRACT_VECTOR_ELT(N); 1192 case ISD::BUILD_VECTOR: return visitBUILD_VECTOR(N); 1193 case ISD::CONCAT_VECTORS: return visitCONCAT_VECTORS(N); 1194 case ISD::EXTRACT_SUBVECTOR: return visitEXTRACT_SUBVECTOR(N); 1195 case ISD::VECTOR_SHUFFLE: return visitVECTOR_SHUFFLE(N); 1196 } 1197 return SDValue(); 1198} 1199 1200SDValue DAGCombiner::combine(SDNode *N) { 1201 SDValue RV = visit(N); 1202 1203 // If nothing happened, try a target-specific DAG combine. 1204 if (RV.getNode() == 0) { 1205 assert(N->getOpcode() != ISD::DELETED_NODE && 1206 "Node was deleted but visit returned NULL!"); 1207 1208 if (N->getOpcode() >= ISD::BUILTIN_OP_END || 1209 TLI.hasTargetDAGCombine((ISD::NodeType)N->getOpcode())) { 1210 1211 // Expose the DAG combiner to the target combiner impls. 1212 TargetLowering::DAGCombinerInfo 1213 DagCombineInfo(DAG, Level, false, this); 1214 1215 RV = TLI.PerformDAGCombine(N, DagCombineInfo); 1216 } 1217 } 1218 1219 // If nothing happened still, try promoting the operation. 1220 if (RV.getNode() == 0) { 1221 switch (N->getOpcode()) { 1222 default: break; 1223 case ISD::ADD: 1224 case ISD::SUB: 1225 case ISD::MUL: 1226 case ISD::AND: 1227 case ISD::OR: 1228 case ISD::XOR: 1229 RV = PromoteIntBinOp(SDValue(N, 0)); 1230 break; 1231 case ISD::SHL: 1232 case ISD::SRA: 1233 case ISD::SRL: 1234 RV = PromoteIntShiftOp(SDValue(N, 0)); 1235 break; 1236 case ISD::SIGN_EXTEND: 1237 case ISD::ZERO_EXTEND: 1238 case ISD::ANY_EXTEND: 1239 RV = PromoteExtend(SDValue(N, 0)); 1240 break; 1241 case ISD::LOAD: 1242 if (PromoteLoad(SDValue(N, 0))) 1243 RV = SDValue(N, 0); 1244 break; 1245 } 1246 } 1247 1248 // If N is a commutative binary node, try commuting it to enable more 1249 // sdisel CSE. 1250 if (RV.getNode() == 0 && 1251 SelectionDAG::isCommutativeBinOp(N->getOpcode()) && 1252 N->getNumValues() == 1) { 1253 SDValue N0 = N->getOperand(0); 1254 SDValue N1 = N->getOperand(1); 1255 1256 // Constant operands are canonicalized to RHS. 1257 if (isa<ConstantSDNode>(N0) || !isa<ConstantSDNode>(N1)) { 1258 SDValue Ops[] = { N1, N0 }; 1259 SDNode *CSENode = DAG.getNodeIfExists(N->getOpcode(), N->getVTList(), 1260 Ops, 2); 1261 if (CSENode) 1262 return SDValue(CSENode, 0); 1263 } 1264 } 1265 1266 return RV; 1267} 1268 1269/// getInputChainForNode - Given a node, return its input chain if it has one, 1270/// otherwise return a null sd operand. 1271static SDValue getInputChainForNode(SDNode *N) { 1272 if (unsigned NumOps = N->getNumOperands()) { 1273 if (N->getOperand(0).getValueType() == MVT::Other) 1274 return N->getOperand(0); 1275 if (N->getOperand(NumOps-1).getValueType() == MVT::Other) 1276 return N->getOperand(NumOps-1); 1277 for (unsigned i = 1; i < NumOps-1; ++i) 1278 if (N->getOperand(i).getValueType() == MVT::Other) 1279 return N->getOperand(i); 1280 } 1281 return SDValue(); 1282} 1283 1284SDValue DAGCombiner::visitTokenFactor(SDNode *N) { 1285 // If N has two operands, where one has an input chain equal to the other, 1286 // the 'other' chain is redundant. 1287 if (N->getNumOperands() == 2) { 1288 if (getInputChainForNode(N->getOperand(0).getNode()) == N->getOperand(1)) 1289 return N->getOperand(0); 1290 if (getInputChainForNode(N->getOperand(1).getNode()) == N->getOperand(0)) 1291 return N->getOperand(1); 1292 } 1293 1294 SmallVector<SDNode *, 8> TFs; // List of token factors to visit. 1295 SmallVector<SDValue, 8> Ops; // Ops for replacing token factor. 1296 SmallPtrSet<SDNode*, 16> SeenOps; 1297 bool Changed = false; // If we should replace this token factor. 1298 1299 // Start out with this token factor. 1300 TFs.push_back(N); 1301 1302 // Iterate through token factors. The TFs grows when new token factors are 1303 // encountered. 1304 for (unsigned i = 0; i < TFs.size(); ++i) { 1305 SDNode *TF = TFs[i]; 1306 1307 // Check each of the operands. 1308 for (unsigned i = 0, ie = TF->getNumOperands(); i != ie; ++i) { 1309 SDValue Op = TF->getOperand(i); 1310 1311 switch (Op.getOpcode()) { 1312 case ISD::EntryToken: 1313 // Entry tokens don't need to be added to the list. They are 1314 // rededundant. 1315 Changed = true; 1316 break; 1317 1318 case ISD::TokenFactor: 1319 if (Op.hasOneUse() && 1320 std::find(TFs.begin(), TFs.end(), Op.getNode()) == TFs.end()) { 1321 // Q…
Large files files are truncated, but you can click here to view the full file