/3rd_party/llvm/lib/CodeGen/SelectionDAG/DAGCombiner.cpp

https://code.google.com/p/softart/ · C++ · 11051 lines · 7632 code · 1358 blank · 2061 comment · 3191 complexity · 9168f24fe989eb6a6817a65e823725b7 MD5 · raw file

Large files are truncated click here to view the full file

  1. //===-- DAGCombiner.cpp - Implement a DAG node combiner -------------------===//
  2. //
  3. // The LLVM Compiler Infrastructure
  4. //
  5. // This file is distributed under the University of Illinois Open Source
  6. // License. See LICENSE.TXT for details.
  7. //
  8. //===----------------------------------------------------------------------===//
  9. //
  10. // This pass combines dag nodes to form fewer, simpler DAG nodes. It can be run
  11. // both before and after the DAG is legalized.
  12. //
  13. // This pass is not a substitute for the LLVM IR instcombine pass. This pass is
  14. // primarily intended to handle simplification opportunities that are implicit
  15. // in the LLVM IR and exposed by the various codegen lowering phases.
  16. //
  17. //===----------------------------------------------------------------------===//
  18. #define DEBUG_TYPE "dagcombine"
  19. #include "llvm/CodeGen/SelectionDAG.h"
  20. #include "llvm/ADT/SmallPtrSet.h"
  21. #include "llvm/ADT/Statistic.h"
  22. #include "llvm/Analysis/AliasAnalysis.h"
  23. #include "llvm/CodeGen/MachineFrameInfo.h"
  24. #include "llvm/CodeGen/MachineFunction.h"
  25. #include "llvm/IR/DataLayout.h"
  26. #include "llvm/IR/DerivedTypes.h"
  27. #include "llvm/IR/Function.h"
  28. #include "llvm/IR/LLVMContext.h"
  29. #include "llvm/Support/CommandLine.h"
  30. #include "llvm/Support/Debug.h"
  31. #include "llvm/Support/ErrorHandling.h"
  32. #include "llvm/Support/MathExtras.h"
  33. #include "llvm/Support/raw_ostream.h"
  34. #include "llvm/Target/TargetLowering.h"
  35. #include "llvm/Target/TargetMachine.h"
  36. #include "llvm/Target/TargetOptions.h"
  37. #include "llvm/Target/TargetRegisterInfo.h"
  38. #include "llvm/Target/TargetSubtargetInfo.h"
  39. #include <algorithm>
  40. using namespace llvm;
  41. STATISTIC(NodesCombined , "Number of dag nodes combined");
  42. STATISTIC(PreIndexedNodes , "Number of pre-indexed nodes created");
  43. STATISTIC(PostIndexedNodes, "Number of post-indexed nodes created");
  44. STATISTIC(OpsNarrowed , "Number of load/op/store narrowed");
  45. STATISTIC(LdStFP2Int , "Number of fp load/store pairs transformed to int");
  46. STATISTIC(SlicedLoads, "Number of load sliced");
  47. namespace {
  48. static cl::opt<bool>
  49. CombinerAA("combiner-alias-analysis", cl::Hidden,
  50. cl::desc("Turn on alias analysis during testing"));
  51. static cl::opt<bool>
  52. CombinerGlobalAA("combiner-global-alias-analysis", cl::Hidden,
  53. cl::desc("Include global information in alias analysis"));
  54. /// Hidden option to stress test load slicing, i.e., when this option
  55. /// is enabled, load slicing bypasses most of its profitability guards.
  56. static cl::opt<bool>
  57. StressLoadSlicing("combiner-stress-load-slicing", cl::Hidden,
  58. cl::desc("Bypass the profitability model of load "
  59. "slicing"),
  60. cl::init(false));
  61. //------------------------------ DAGCombiner ---------------------------------//
  62. class DAGCombiner {
  63. SelectionDAG &DAG;
  64. const TargetLowering &TLI;
  65. CombineLevel Level;
  66. CodeGenOpt::Level OptLevel;
  67. bool LegalOperations;
  68. bool LegalTypes;
  69. bool ForCodeSize;
  70. // Worklist of all of the nodes that need to be simplified.
  71. //
  72. // This has the semantics that when adding to the worklist,
  73. // the item added must be next to be processed. It should
  74. // also only appear once. The naive approach to this takes
  75. // linear time.
  76. //
  77. // To reduce the insert/remove time to logarithmic, we use
  78. // a set and a vector to maintain our worklist.
  79. //
  80. // The set contains the items on the worklist, but does not
  81. // maintain the order they should be visited.
  82. //
  83. // The vector maintains the order nodes should be visited, but may
  84. // contain duplicate or removed nodes. When choosing a node to
  85. // visit, we pop off the order stack until we find an item that is
  86. // also in the contents set. All operations are O(log N).
  87. SmallPtrSet<SDNode*, 64> WorkListContents;
  88. SmallVector<SDNode*, 64> WorkListOrder;
  89. // AA - Used for DAG load/store alias analysis.
  90. AliasAnalysis &AA;
  91. /// AddUsersToWorkList - When an instruction is simplified, add all users of
  92. /// the instruction to the work lists because they might get more simplified
  93. /// now.
  94. ///
  95. void AddUsersToWorkList(SDNode *N) {
  96. for (SDNode::use_iterator UI = N->use_begin(), UE = N->use_end();
  97. UI != UE; ++UI)
  98. AddToWorkList(*UI);
  99. }
  100. /// visit - call the node-specific routine that knows how to fold each
  101. /// particular type of node.
  102. SDValue visit(SDNode *N);
  103. public:
  104. /// AddToWorkList - Add to the work list making sure its instance is at the
  105. /// back (next to be processed.)
  106. void AddToWorkList(SDNode *N) {
  107. WorkListContents.insert(N);
  108. WorkListOrder.push_back(N);
  109. }
  110. /// removeFromWorkList - remove all instances of N from the worklist.
  111. ///
  112. void removeFromWorkList(SDNode *N) {
  113. WorkListContents.erase(N);
  114. }
  115. SDValue CombineTo(SDNode *N, const SDValue *To, unsigned NumTo,
  116. bool AddTo = true);
  117. SDValue CombineTo(SDNode *N, SDValue Res, bool AddTo = true) {
  118. return CombineTo(N, &Res, 1, AddTo);
  119. }
  120. SDValue CombineTo(SDNode *N, SDValue Res0, SDValue Res1,
  121. bool AddTo = true) {
  122. SDValue To[] = { Res0, Res1 };
  123. return CombineTo(N, To, 2, AddTo);
  124. }
  125. void CommitTargetLoweringOpt(const TargetLowering::TargetLoweringOpt &TLO);
  126. private:
  127. /// SimplifyDemandedBits - Check the specified integer node value to see if
  128. /// it can be simplified or if things it uses can be simplified by bit
  129. /// propagation. If so, return true.
  130. bool SimplifyDemandedBits(SDValue Op) {
  131. unsigned BitWidth = Op.getValueType().getScalarType().getSizeInBits();
  132. APInt Demanded = APInt::getAllOnesValue(BitWidth);
  133. return SimplifyDemandedBits(Op, Demanded);
  134. }
  135. bool SimplifyDemandedBits(SDValue Op, const APInt &Demanded);
  136. bool CombineToPreIndexedLoadStore(SDNode *N);
  137. bool CombineToPostIndexedLoadStore(SDNode *N);
  138. bool SliceUpLoad(SDNode *N);
  139. void ReplaceLoadWithPromotedLoad(SDNode *Load, SDNode *ExtLoad);
  140. SDValue PromoteOperand(SDValue Op, EVT PVT, bool &Replace);
  141. SDValue SExtPromoteOperand(SDValue Op, EVT PVT);
  142. SDValue ZExtPromoteOperand(SDValue Op, EVT PVT);
  143. SDValue PromoteIntBinOp(SDValue Op);
  144. SDValue PromoteIntShiftOp(SDValue Op);
  145. SDValue PromoteExtend(SDValue Op);
  146. bool PromoteLoad(SDValue Op);
  147. void ExtendSetCCUses(const SmallVectorImpl<SDNode *> &SetCCs,
  148. SDValue Trunc, SDValue ExtLoad, SDLoc DL,
  149. ISD::NodeType ExtType);
  150. /// combine - call the node-specific routine that knows how to fold each
  151. /// particular type of node. If that doesn't do anything, try the
  152. /// target-specific DAG combines.
  153. SDValue combine(SDNode *N);
  154. // Visitation implementation - Implement dag node combining for different
  155. // node types. The semantics are as follows:
  156. // Return Value:
  157. // SDValue.getNode() == 0 - No change was made
  158. // SDValue.getNode() == N - N was replaced, is dead and has been handled.
  159. // otherwise - N should be replaced by the returned Operand.
  160. //
  161. SDValue visitTokenFactor(SDNode *N);
  162. SDValue visitMERGE_VALUES(SDNode *N);
  163. SDValue visitADD(SDNode *N);
  164. SDValue visitSUB(SDNode *N);
  165. SDValue visitADDC(SDNode *N);
  166. SDValue visitSUBC(SDNode *N);
  167. SDValue visitADDE(SDNode *N);
  168. SDValue visitSUBE(SDNode *N);
  169. SDValue visitMUL(SDNode *N);
  170. SDValue visitSDIV(SDNode *N);
  171. SDValue visitUDIV(SDNode *N);
  172. SDValue visitSREM(SDNode *N);
  173. SDValue visitUREM(SDNode *N);
  174. SDValue visitMULHU(SDNode *N);
  175. SDValue visitMULHS(SDNode *N);
  176. SDValue visitSMUL_LOHI(SDNode *N);
  177. SDValue visitUMUL_LOHI(SDNode *N);
  178. SDValue visitSMULO(SDNode *N);
  179. SDValue visitUMULO(SDNode *N);
  180. SDValue visitSDIVREM(SDNode *N);
  181. SDValue visitUDIVREM(SDNode *N);
  182. SDValue visitAND(SDNode *N);
  183. SDValue visitOR(SDNode *N);
  184. SDValue visitXOR(SDNode *N);
  185. SDValue SimplifyVBinOp(SDNode *N);
  186. SDValue SimplifyVUnaryOp(SDNode *N);
  187. SDValue visitSHL(SDNode *N);
  188. SDValue visitSRA(SDNode *N);
  189. SDValue visitSRL(SDNode *N);
  190. SDValue visitCTLZ(SDNode *N);
  191. SDValue visitCTLZ_ZERO_UNDEF(SDNode *N);
  192. SDValue visitCTTZ(SDNode *N);
  193. SDValue visitCTTZ_ZERO_UNDEF(SDNode *N);
  194. SDValue visitCTPOP(SDNode *N);
  195. SDValue visitSELECT(SDNode *N);
  196. SDValue visitVSELECT(SDNode *N);
  197. SDValue visitSELECT_CC(SDNode *N);
  198. SDValue visitSETCC(SDNode *N);
  199. SDValue visitSIGN_EXTEND(SDNode *N);
  200. SDValue visitZERO_EXTEND(SDNode *N);
  201. SDValue visitANY_EXTEND(SDNode *N);
  202. SDValue visitSIGN_EXTEND_INREG(SDNode *N);
  203. SDValue visitTRUNCATE(SDNode *N);
  204. SDValue visitBITCAST(SDNode *N);
  205. SDValue visitBUILD_PAIR(SDNode *N);
  206. SDValue visitFADD(SDNode *N);
  207. SDValue visitFSUB(SDNode *N);
  208. SDValue visitFMUL(SDNode *N);
  209. SDValue visitFMA(SDNode *N);
  210. SDValue visitFDIV(SDNode *N);
  211. SDValue visitFREM(SDNode *N);
  212. SDValue visitFCOPYSIGN(SDNode *N);
  213. SDValue visitSINT_TO_FP(SDNode *N);
  214. SDValue visitUINT_TO_FP(SDNode *N);
  215. SDValue visitFP_TO_SINT(SDNode *N);
  216. SDValue visitFP_TO_UINT(SDNode *N);
  217. SDValue visitFP_ROUND(SDNode *N);
  218. SDValue visitFP_ROUND_INREG(SDNode *N);
  219. SDValue visitFP_EXTEND(SDNode *N);
  220. SDValue visitFNEG(SDNode *N);
  221. SDValue visitFABS(SDNode *N);
  222. SDValue visitFCEIL(SDNode *N);
  223. SDValue visitFTRUNC(SDNode *N);
  224. SDValue visitFFLOOR(SDNode *N);
  225. SDValue visitBRCOND(SDNode *N);
  226. SDValue visitBR_CC(SDNode *N);
  227. SDValue visitLOAD(SDNode *N);
  228. SDValue visitSTORE(SDNode *N);
  229. SDValue visitINSERT_VECTOR_ELT(SDNode *N);
  230. SDValue visitEXTRACT_VECTOR_ELT(SDNode *N);
  231. SDValue visitBUILD_VECTOR(SDNode *N);
  232. SDValue visitCONCAT_VECTORS(SDNode *N);
  233. SDValue visitEXTRACT_SUBVECTOR(SDNode *N);
  234. SDValue visitVECTOR_SHUFFLE(SDNode *N);
  235. SDValue XformToShuffleWithZero(SDNode *N);
  236. SDValue ReassociateOps(unsigned Opc, SDLoc DL, SDValue LHS, SDValue RHS);
  237. SDValue visitShiftByConstant(SDNode *N, unsigned Amt);
  238. bool SimplifySelectOps(SDNode *SELECT, SDValue LHS, SDValue RHS);
  239. SDValue SimplifyBinOpWithSameOpcodeHands(SDNode *N);
  240. SDValue SimplifySelect(SDLoc DL, SDValue N0, SDValue N1, SDValue N2);
  241. SDValue SimplifySelectCC(SDLoc DL, SDValue N0, SDValue N1, SDValue N2,
  242. SDValue N3, ISD::CondCode CC,
  243. bool NotExtCompare = false);
  244. SDValue SimplifySetCC(EVT VT, SDValue N0, SDValue N1, ISD::CondCode Cond,
  245. SDLoc DL, bool foldBooleans = true);
  246. SDValue SimplifyNodeWithTwoResults(SDNode *N, unsigned LoOp,
  247. unsigned HiOp);
  248. SDValue CombineConsecutiveLoads(SDNode *N, EVT VT);
  249. SDValue ConstantFoldBITCASTofBUILD_VECTOR(SDNode *, EVT);
  250. SDValue BuildSDIV(SDNode *N);
  251. SDValue BuildUDIV(SDNode *N);
  252. SDValue MatchBSwapHWordLow(SDNode *N, SDValue N0, SDValue N1,
  253. bool DemandHighBits = true);
  254. SDValue MatchBSwapHWord(SDNode *N, SDValue N0, SDValue N1);
  255. SDNode *MatchRotate(SDValue LHS, SDValue RHS, SDLoc DL);
  256. SDValue ReduceLoadWidth(SDNode *N);
  257. SDValue ReduceLoadOpStoreWidth(SDNode *N);
  258. SDValue TransformFPLoadStorePair(SDNode *N);
  259. SDValue reduceBuildVecExtToExtBuildVec(SDNode *N);
  260. SDValue reduceBuildVecConvertToConvertBuildVec(SDNode *N);
  261. SDValue GetDemandedBits(SDValue V, const APInt &Mask);
  262. /// GatherAllAliases - Walk up chain skipping non-aliasing memory nodes,
  263. /// looking for aliasing nodes and adding them to the Aliases vector.
  264. void GatherAllAliases(SDNode *N, SDValue OriginalChain,
  265. SmallVectorImpl<SDValue> &Aliases);
  266. /// isAlias - Return true if there is any possibility that the two addresses
  267. /// overlap.
  268. bool isAlias(SDValue Ptr1, int64_t Size1, bool IsVolatile1,
  269. const Value *SrcValue1, int SrcValueOffset1,
  270. unsigned SrcValueAlign1,
  271. const MDNode *TBAAInfo1,
  272. SDValue Ptr2, int64_t Size2, bool IsVolatile2,
  273. const Value *SrcValue2, int SrcValueOffset2,
  274. unsigned SrcValueAlign2,
  275. const MDNode *TBAAInfo2) const;
  276. /// isAlias - Return true if there is any possibility that the two addresses
  277. /// overlap.
  278. bool isAlias(LSBaseSDNode *Op0, LSBaseSDNode *Op1);
  279. /// FindAliasInfo - Extracts the relevant alias information from the memory
  280. /// node. Returns true if the operand was a load.
  281. bool FindAliasInfo(SDNode *N,
  282. SDValue &Ptr, int64_t &Size, bool &IsVolatile,
  283. const Value *&SrcValue, int &SrcValueOffset,
  284. unsigned &SrcValueAlignment,
  285. const MDNode *&TBAAInfo) const;
  286. /// FindBetterChain - Walk up chain skipping non-aliasing memory nodes,
  287. /// looking for a better chain (aliasing node.)
  288. SDValue FindBetterChain(SDNode *N, SDValue Chain);
  289. /// Merge consecutive store operations into a wide store.
  290. /// This optimization uses wide integers or vectors when possible.
  291. /// \return True if some memory operations were changed.
  292. bool MergeConsecutiveStores(StoreSDNode *N);
  293. public:
  294. DAGCombiner(SelectionDAG &D, AliasAnalysis &A, CodeGenOpt::Level OL)
  295. : DAG(D), TLI(D.getTargetLoweringInfo()), Level(BeforeLegalizeTypes),
  296. OptLevel(OL), LegalOperations(false), LegalTypes(false), AA(A) {
  297. AttributeSet FnAttrs =
  298. DAG.getMachineFunction().getFunction()->getAttributes();
  299. ForCodeSize =
  300. FnAttrs.hasAttribute(AttributeSet::FunctionIndex,
  301. Attribute::OptimizeForSize) ||
  302. FnAttrs.hasAttribute(AttributeSet::FunctionIndex, Attribute::MinSize);
  303. }
  304. /// Run - runs the dag combiner on all nodes in the work list
  305. void Run(CombineLevel AtLevel);
  306. SelectionDAG &getDAG() const { return DAG; }
  307. /// getShiftAmountTy - Returns a type large enough to hold any valid
  308. /// shift amount - before type legalization these can be huge.
  309. EVT getShiftAmountTy(EVT LHSTy) {
  310. assert(LHSTy.isInteger() && "Shift amount is not an integer type!");
  311. if (LHSTy.isVector())
  312. return LHSTy;
  313. return LegalTypes ? TLI.getScalarShiftAmountTy(LHSTy)
  314. : TLI.getPointerTy();
  315. }
  316. /// isTypeLegal - This method returns true if we are running before type
  317. /// legalization or if the specified VT is legal.
  318. bool isTypeLegal(const EVT &VT) {
  319. if (!LegalTypes) return true;
  320. return TLI.isTypeLegal(VT);
  321. }
  322. /// getSetCCResultType - Convenience wrapper around
  323. /// TargetLowering::getSetCCResultType
  324. EVT getSetCCResultType(EVT VT) const {
  325. return TLI.getSetCCResultType(*DAG.getContext(), VT);
  326. }
  327. };
  328. }
  329. namespace {
  330. /// WorkListRemover - This class is a DAGUpdateListener that removes any deleted
  331. /// nodes from the worklist.
  332. class WorkListRemover : public SelectionDAG::DAGUpdateListener {
  333. DAGCombiner &DC;
  334. public:
  335. explicit WorkListRemover(DAGCombiner &dc)
  336. : SelectionDAG::DAGUpdateListener(dc.getDAG()), DC(dc) {}
  337. virtual void NodeDeleted(SDNode *N, SDNode *E) {
  338. DC.removeFromWorkList(N);
  339. }
  340. };
  341. }
  342. //===----------------------------------------------------------------------===//
  343. // TargetLowering::DAGCombinerInfo implementation
  344. //===----------------------------------------------------------------------===//
  345. void TargetLowering::DAGCombinerInfo::AddToWorklist(SDNode *N) {
  346. ((DAGCombiner*)DC)->AddToWorkList(N);
  347. }
  348. void TargetLowering::DAGCombinerInfo::RemoveFromWorklist(SDNode *N) {
  349. ((DAGCombiner*)DC)->removeFromWorkList(N);
  350. }
  351. SDValue TargetLowering::DAGCombinerInfo::
  352. CombineTo(SDNode *N, const std::vector<SDValue> &To, bool AddTo) {
  353. return ((DAGCombiner*)DC)->CombineTo(N, &To[0], To.size(), AddTo);
  354. }
  355. SDValue TargetLowering::DAGCombinerInfo::
  356. CombineTo(SDNode *N, SDValue Res, bool AddTo) {
  357. return ((DAGCombiner*)DC)->CombineTo(N, Res, AddTo);
  358. }
  359. SDValue TargetLowering::DAGCombinerInfo::
  360. CombineTo(SDNode *N, SDValue Res0, SDValue Res1, bool AddTo) {
  361. return ((DAGCombiner*)DC)->CombineTo(N, Res0, Res1, AddTo);
  362. }
  363. void TargetLowering::DAGCombinerInfo::
  364. CommitTargetLoweringOpt(const TargetLowering::TargetLoweringOpt &TLO) {
  365. return ((DAGCombiner*)DC)->CommitTargetLoweringOpt(TLO);
  366. }
  367. //===----------------------------------------------------------------------===//
  368. // Helper Functions
  369. //===----------------------------------------------------------------------===//
  370. /// isNegatibleForFree - Return 1 if we can compute the negated form of the
  371. /// specified expression for the same cost as the expression itself, or 2 if we
  372. /// can compute the negated form more cheaply than the expression itself.
  373. static char isNegatibleForFree(SDValue Op, bool LegalOperations,
  374. const TargetLowering &TLI,
  375. const TargetOptions *Options,
  376. unsigned Depth = 0) {
  377. // fneg is removable even if it has multiple uses.
  378. if (Op.getOpcode() == ISD::FNEG) return 2;
  379. // Don't allow anything with multiple uses.
  380. if (!Op.hasOneUse()) return 0;
  381. // Don't recurse exponentially.
  382. if (Depth > 6) return 0;
  383. switch (Op.getOpcode()) {
  384. default: return false;
  385. case ISD::ConstantFP:
  386. // Don't invert constant FP values after legalize. The negated constant
  387. // isn't necessarily legal.
  388. return LegalOperations ? 0 : 1;
  389. case ISD::FADD:
  390. // FIXME: determine better conditions for this xform.
  391. if (!Options->UnsafeFPMath) return 0;
  392. // After operation legalization, it might not be legal to create new FSUBs.
  393. if (LegalOperations &&
  394. !TLI.isOperationLegalOrCustom(ISD::FSUB, Op.getValueType()))
  395. return 0;
  396. // fold (fneg (fadd A, B)) -> (fsub (fneg A), B)
  397. if (char V = isNegatibleForFree(Op.getOperand(0), LegalOperations, TLI,
  398. Options, Depth + 1))
  399. return V;
  400. // fold (fneg (fadd A, B)) -> (fsub (fneg B), A)
  401. return isNegatibleForFree(Op.getOperand(1), LegalOperations, TLI, Options,
  402. Depth + 1);
  403. case ISD::FSUB:
  404. // We can't turn -(A-B) into B-A when we honor signed zeros.
  405. if (!Options->UnsafeFPMath) return 0;
  406. // fold (fneg (fsub A, B)) -> (fsub B, A)
  407. return 1;
  408. case ISD::FMUL:
  409. case ISD::FDIV:
  410. if (Options->HonorSignDependentRoundingFPMath()) return 0;
  411. // fold (fneg (fmul X, Y)) -> (fmul (fneg X), Y) or (fmul X, (fneg Y))
  412. if (char V = isNegatibleForFree(Op.getOperand(0), LegalOperations, TLI,
  413. Options, Depth + 1))
  414. return V;
  415. return isNegatibleForFree(Op.getOperand(1), LegalOperations, TLI, Options,
  416. Depth + 1);
  417. case ISD::FP_EXTEND:
  418. case ISD::FP_ROUND:
  419. case ISD::FSIN:
  420. return isNegatibleForFree(Op.getOperand(0), LegalOperations, TLI, Options,
  421. Depth + 1);
  422. }
  423. }
  424. /// GetNegatedExpression - If isNegatibleForFree returns true, this function
  425. /// returns the newly negated expression.
  426. static SDValue GetNegatedExpression(SDValue Op, SelectionDAG &DAG,
  427. bool LegalOperations, unsigned Depth = 0) {
  428. // fneg is removable even if it has multiple uses.
  429. if (Op.getOpcode() == ISD::FNEG) return Op.getOperand(0);
  430. // Don't allow anything with multiple uses.
  431. assert(Op.hasOneUse() && "Unknown reuse!");
  432. assert(Depth <= 6 && "GetNegatedExpression doesn't match isNegatibleForFree");
  433. switch (Op.getOpcode()) {
  434. default: llvm_unreachable("Unknown code");
  435. case ISD::ConstantFP: {
  436. APFloat V = cast<ConstantFPSDNode>(Op)->getValueAPF();
  437. V.changeSign();
  438. return DAG.getConstantFP(V, Op.getValueType());
  439. }
  440. case ISD::FADD:
  441. // FIXME: determine better conditions for this xform.
  442. assert(DAG.getTarget().Options.UnsafeFPMath);
  443. // fold (fneg (fadd A, B)) -> (fsub (fneg A), B)
  444. if (isNegatibleForFree(Op.getOperand(0), LegalOperations,
  445. DAG.getTargetLoweringInfo(),
  446. &DAG.getTarget().Options, Depth+1))
  447. return DAG.getNode(ISD::FSUB, SDLoc(Op), Op.getValueType(),
  448. GetNegatedExpression(Op.getOperand(0), DAG,
  449. LegalOperations, Depth+1),
  450. Op.getOperand(1));
  451. // fold (fneg (fadd A, B)) -> (fsub (fneg B), A)
  452. return DAG.getNode(ISD::FSUB, SDLoc(Op), Op.getValueType(),
  453. GetNegatedExpression(Op.getOperand(1), DAG,
  454. LegalOperations, Depth+1),
  455. Op.getOperand(0));
  456. case ISD::FSUB:
  457. // We can't turn -(A-B) into B-A when we honor signed zeros.
  458. assert(DAG.getTarget().Options.UnsafeFPMath);
  459. // fold (fneg (fsub 0, B)) -> B
  460. if (ConstantFPSDNode *N0CFP = dyn_cast<ConstantFPSDNode>(Op.getOperand(0)))
  461. if (N0CFP->getValueAPF().isZero())
  462. return Op.getOperand(1);
  463. // fold (fneg (fsub A, B)) -> (fsub B, A)
  464. return DAG.getNode(ISD::FSUB, SDLoc(Op), Op.getValueType(),
  465. Op.getOperand(1), Op.getOperand(0));
  466. case ISD::FMUL:
  467. case ISD::FDIV:
  468. assert(!DAG.getTarget().Options.HonorSignDependentRoundingFPMath());
  469. // fold (fneg (fmul X, Y)) -> (fmul (fneg X), Y)
  470. if (isNegatibleForFree(Op.getOperand(0), LegalOperations,
  471. DAG.getTargetLoweringInfo(),
  472. &DAG.getTarget().Options, Depth+1))
  473. return DAG.getNode(Op.getOpcode(), SDLoc(Op), Op.getValueType(),
  474. GetNegatedExpression(Op.getOperand(0), DAG,
  475. LegalOperations, Depth+1),
  476. Op.getOperand(1));
  477. // fold (fneg (fmul X, Y)) -> (fmul X, (fneg Y))
  478. return DAG.getNode(Op.getOpcode(), SDLoc(Op), Op.getValueType(),
  479. Op.getOperand(0),
  480. GetNegatedExpression(Op.getOperand(1), DAG,
  481. LegalOperations, Depth+1));
  482. case ISD::FP_EXTEND:
  483. case ISD::FSIN:
  484. return DAG.getNode(Op.getOpcode(), SDLoc(Op), Op.getValueType(),
  485. GetNegatedExpression(Op.getOperand(0), DAG,
  486. LegalOperations, Depth+1));
  487. case ISD::FP_ROUND:
  488. return DAG.getNode(ISD::FP_ROUND, SDLoc(Op), Op.getValueType(),
  489. GetNegatedExpression(Op.getOperand(0), DAG,
  490. LegalOperations, Depth+1),
  491. Op.getOperand(1));
  492. }
  493. }
  494. // isSetCCEquivalent - Return true if this node is a setcc, or is a select_cc
  495. // that selects between the values 1 and 0, making it equivalent to a setcc.
  496. // Also, set the incoming LHS, RHS, and CC references to the appropriate
  497. // nodes based on the type of node we are checking. This simplifies life a
  498. // bit for the callers.
  499. static bool isSetCCEquivalent(SDValue N, SDValue &LHS, SDValue &RHS,
  500. SDValue &CC) {
  501. if (N.getOpcode() == ISD::SETCC) {
  502. LHS = N.getOperand(0);
  503. RHS = N.getOperand(1);
  504. CC = N.getOperand(2);
  505. return true;
  506. }
  507. if (N.getOpcode() == ISD::SELECT_CC &&
  508. N.getOperand(2).getOpcode() == ISD::Constant &&
  509. N.getOperand(3).getOpcode() == ISD::Constant &&
  510. cast<ConstantSDNode>(N.getOperand(2))->getAPIntValue() == 1 &&
  511. cast<ConstantSDNode>(N.getOperand(3))->isNullValue()) {
  512. LHS = N.getOperand(0);
  513. RHS = N.getOperand(1);
  514. CC = N.getOperand(4);
  515. return true;
  516. }
  517. return false;
  518. }
  519. // isOneUseSetCC - Return true if this is a SetCC-equivalent operation with only
  520. // one use. If this is true, it allows the users to invert the operation for
  521. // free when it is profitable to do so.
  522. static bool isOneUseSetCC(SDValue N) {
  523. SDValue N0, N1, N2;
  524. if (isSetCCEquivalent(N, N0, N1, N2) && N.getNode()->hasOneUse())
  525. return true;
  526. return false;
  527. }
  528. SDValue DAGCombiner::ReassociateOps(unsigned Opc, SDLoc DL,
  529. SDValue N0, SDValue N1) {
  530. EVT VT = N0.getValueType();
  531. if (N0.getOpcode() == Opc && isa<ConstantSDNode>(N0.getOperand(1))) {
  532. if (isa<ConstantSDNode>(N1)) {
  533. // reassoc. (op (op x, c1), c2) -> (op x, (op c1, c2))
  534. SDValue OpNode =
  535. DAG.FoldConstantArithmetic(Opc, VT,
  536. cast<ConstantSDNode>(N0.getOperand(1)),
  537. cast<ConstantSDNode>(N1));
  538. return DAG.getNode(Opc, DL, VT, N0.getOperand(0), OpNode);
  539. }
  540. if (N0.hasOneUse()) {
  541. // reassoc. (op (op x, c1), y) -> (op (op x, y), c1) iff x+c1 has one use
  542. SDValue OpNode = DAG.getNode(Opc, SDLoc(N0), VT,
  543. N0.getOperand(0), N1);
  544. AddToWorkList(OpNode.getNode());
  545. return DAG.getNode(Opc, DL, VT, OpNode, N0.getOperand(1));
  546. }
  547. }
  548. if (N1.getOpcode() == Opc && isa<ConstantSDNode>(N1.getOperand(1))) {
  549. if (isa<ConstantSDNode>(N0)) {
  550. // reassoc. (op c2, (op x, c1)) -> (op x, (op c1, c2))
  551. SDValue OpNode =
  552. DAG.FoldConstantArithmetic(Opc, VT,
  553. cast<ConstantSDNode>(N1.getOperand(1)),
  554. cast<ConstantSDNode>(N0));
  555. return DAG.getNode(Opc, DL, VT, N1.getOperand(0), OpNode);
  556. }
  557. if (N1.hasOneUse()) {
  558. // reassoc. (op y, (op x, c1)) -> (op (op x, y), c1) iff x+c1 has one use
  559. SDValue OpNode = DAG.getNode(Opc, SDLoc(N0), VT,
  560. N1.getOperand(0), N0);
  561. AddToWorkList(OpNode.getNode());
  562. return DAG.getNode(Opc, DL, VT, OpNode, N1.getOperand(1));
  563. }
  564. }
  565. return SDValue();
  566. }
  567. SDValue DAGCombiner::CombineTo(SDNode *N, const SDValue *To, unsigned NumTo,
  568. bool AddTo) {
  569. assert(N->getNumValues() == NumTo && "Broken CombineTo call!");
  570. ++NodesCombined;
  571. DEBUG(dbgs() << "\nReplacing.1 ";
  572. N->dump(&DAG);
  573. dbgs() << "\nWith: ";
  574. To[0].getNode()->dump(&DAG);
  575. dbgs() << " and " << NumTo-1 << " other values\n";
  576. for (unsigned i = 0, e = NumTo; i != e; ++i)
  577. assert((!To[i].getNode() ||
  578. N->getValueType(i) == To[i].getValueType()) &&
  579. "Cannot combine value to value of different type!"));
  580. WorkListRemover DeadNodes(*this);
  581. DAG.ReplaceAllUsesWith(N, To);
  582. if (AddTo) {
  583. // Push the new nodes and any users onto the worklist
  584. for (unsigned i = 0, e = NumTo; i != e; ++i) {
  585. if (To[i].getNode()) {
  586. AddToWorkList(To[i].getNode());
  587. AddUsersToWorkList(To[i].getNode());
  588. }
  589. }
  590. }
  591. // Finally, if the node is now dead, remove it from the graph. The node
  592. // may not be dead if the replacement process recursively simplified to
  593. // something else needing this node.
  594. if (N->use_empty()) {
  595. // Nodes can be reintroduced into the worklist. Make sure we do not
  596. // process a node that has been replaced.
  597. removeFromWorkList(N);
  598. // Finally, since the node is now dead, remove it from the graph.
  599. DAG.DeleteNode(N);
  600. }
  601. return SDValue(N, 0);
  602. }
  603. void DAGCombiner::
  604. CommitTargetLoweringOpt(const TargetLowering::TargetLoweringOpt &TLO) {
  605. // Replace all uses. If any nodes become isomorphic to other nodes and
  606. // are deleted, make sure to remove them from our worklist.
  607. WorkListRemover DeadNodes(*this);
  608. DAG.ReplaceAllUsesOfValueWith(TLO.Old, TLO.New);
  609. // Push the new node and any (possibly new) users onto the worklist.
  610. AddToWorkList(TLO.New.getNode());
  611. AddUsersToWorkList(TLO.New.getNode());
  612. // Finally, if the node is now dead, remove it from the graph. The node
  613. // may not be dead if the replacement process recursively simplified to
  614. // something else needing this node.
  615. if (TLO.Old.getNode()->use_empty()) {
  616. removeFromWorkList(TLO.Old.getNode());
  617. // If the operands of this node are only used by the node, they will now
  618. // be dead. Make sure to visit them first to delete dead nodes early.
  619. for (unsigned i = 0, e = TLO.Old.getNode()->getNumOperands(); i != e; ++i)
  620. if (TLO.Old.getNode()->getOperand(i).getNode()->hasOneUse())
  621. AddToWorkList(TLO.Old.getNode()->getOperand(i).getNode());
  622. DAG.DeleteNode(TLO.Old.getNode());
  623. }
  624. }
  625. /// SimplifyDemandedBits - Check the specified integer node value to see if
  626. /// it can be simplified or if things it uses can be simplified by bit
  627. /// propagation. If so, return true.
  628. bool DAGCombiner::SimplifyDemandedBits(SDValue Op, const APInt &Demanded) {
  629. TargetLowering::TargetLoweringOpt TLO(DAG, LegalTypes, LegalOperations);
  630. APInt KnownZero, KnownOne;
  631. if (!TLI.SimplifyDemandedBits(Op, Demanded, KnownZero, KnownOne, TLO))
  632. return false;
  633. // Revisit the node.
  634. AddToWorkList(Op.getNode());
  635. // Replace the old value with the new one.
  636. ++NodesCombined;
  637. DEBUG(dbgs() << "\nReplacing.2 ";
  638. TLO.Old.getNode()->dump(&DAG);
  639. dbgs() << "\nWith: ";
  640. TLO.New.getNode()->dump(&DAG);
  641. dbgs() << '\n');
  642. CommitTargetLoweringOpt(TLO);
  643. return true;
  644. }
  645. void DAGCombiner::ReplaceLoadWithPromotedLoad(SDNode *Load, SDNode *ExtLoad) {
  646. SDLoc dl(Load);
  647. EVT VT = Load->getValueType(0);
  648. SDValue Trunc = DAG.getNode(ISD::TRUNCATE, dl, VT, SDValue(ExtLoad, 0));
  649. DEBUG(dbgs() << "\nReplacing.9 ";
  650. Load->dump(&DAG);
  651. dbgs() << "\nWith: ";
  652. Trunc.getNode()->dump(&DAG);
  653. dbgs() << '\n');
  654. WorkListRemover DeadNodes(*this);
  655. DAG.ReplaceAllUsesOfValueWith(SDValue(Load, 0), Trunc);
  656. DAG.ReplaceAllUsesOfValueWith(SDValue(Load, 1), SDValue(ExtLoad, 1));
  657. removeFromWorkList(Load);
  658. DAG.DeleteNode(Load);
  659. AddToWorkList(Trunc.getNode());
  660. }
  661. SDValue DAGCombiner::PromoteOperand(SDValue Op, EVT PVT, bool &Replace) {
  662. Replace = false;
  663. SDLoc dl(Op);
  664. if (LoadSDNode *LD = dyn_cast<LoadSDNode>(Op)) {
  665. EVT MemVT = LD->getMemoryVT();
  666. ISD::LoadExtType ExtType = ISD::isNON_EXTLoad(LD)
  667. ? (TLI.isLoadExtLegal(ISD::ZEXTLOAD, MemVT) ? ISD::ZEXTLOAD
  668. : ISD::EXTLOAD)
  669. : LD->getExtensionType();
  670. Replace = true;
  671. return DAG.getExtLoad(ExtType, dl, PVT,
  672. LD->getChain(), LD->getBasePtr(),
  673. MemVT, LD->getMemOperand());
  674. }
  675. unsigned Opc = Op.getOpcode();
  676. switch (Opc) {
  677. default: break;
  678. case ISD::AssertSext:
  679. return DAG.getNode(ISD::AssertSext, dl, PVT,
  680. SExtPromoteOperand(Op.getOperand(0), PVT),
  681. Op.getOperand(1));
  682. case ISD::AssertZext:
  683. return DAG.getNode(ISD::AssertZext, dl, PVT,
  684. ZExtPromoteOperand(Op.getOperand(0), PVT),
  685. Op.getOperand(1));
  686. case ISD::Constant: {
  687. unsigned ExtOpc =
  688. Op.getValueType().isByteSized() ? ISD::SIGN_EXTEND : ISD::ZERO_EXTEND;
  689. return DAG.getNode(ExtOpc, dl, PVT, Op);
  690. }
  691. }
  692. if (!TLI.isOperationLegal(ISD::ANY_EXTEND, PVT))
  693. return SDValue();
  694. return DAG.getNode(ISD::ANY_EXTEND, dl, PVT, Op);
  695. }
  696. SDValue DAGCombiner::SExtPromoteOperand(SDValue Op, EVT PVT) {
  697. if (!TLI.isOperationLegal(ISD::SIGN_EXTEND_INREG, PVT))
  698. return SDValue();
  699. EVT OldVT = Op.getValueType();
  700. SDLoc dl(Op);
  701. bool Replace = false;
  702. SDValue NewOp = PromoteOperand(Op, PVT, Replace);
  703. if (NewOp.getNode() == 0)
  704. return SDValue();
  705. AddToWorkList(NewOp.getNode());
  706. if (Replace)
  707. ReplaceLoadWithPromotedLoad(Op.getNode(), NewOp.getNode());
  708. return DAG.getNode(ISD::SIGN_EXTEND_INREG, dl, NewOp.getValueType(), NewOp,
  709. DAG.getValueType(OldVT));
  710. }
  711. SDValue DAGCombiner::ZExtPromoteOperand(SDValue Op, EVT PVT) {
  712. EVT OldVT = Op.getValueType();
  713. SDLoc dl(Op);
  714. bool Replace = false;
  715. SDValue NewOp = PromoteOperand(Op, PVT, Replace);
  716. if (NewOp.getNode() == 0)
  717. return SDValue();
  718. AddToWorkList(NewOp.getNode());
  719. if (Replace)
  720. ReplaceLoadWithPromotedLoad(Op.getNode(), NewOp.getNode());
  721. return DAG.getZeroExtendInReg(NewOp, dl, OldVT);
  722. }
  723. /// PromoteIntBinOp - Promote the specified integer binary operation if the
  724. /// target indicates it is beneficial. e.g. On x86, it's usually better to
  725. /// promote i16 operations to i32 since i16 instructions are longer.
  726. SDValue DAGCombiner::PromoteIntBinOp(SDValue Op) {
  727. if (!LegalOperations)
  728. return SDValue();
  729. EVT VT = Op.getValueType();
  730. if (VT.isVector() || !VT.isInteger())
  731. return SDValue();
  732. // If operation type is 'undesirable', e.g. i16 on x86, consider
  733. // promoting it.
  734. unsigned Opc = Op.getOpcode();
  735. if (TLI.isTypeDesirableForOp(Opc, VT))
  736. return SDValue();
  737. EVT PVT = VT;
  738. // Consult target whether it is a good idea to promote this operation and
  739. // what's the right type to promote it to.
  740. if (TLI.IsDesirableToPromoteOp(Op, PVT)) {
  741. assert(PVT != VT && "Don't know what type to promote to!");
  742. bool Replace0 = false;
  743. SDValue N0 = Op.getOperand(0);
  744. SDValue NN0 = PromoteOperand(N0, PVT, Replace0);
  745. if (NN0.getNode() == 0)
  746. return SDValue();
  747. bool Replace1 = false;
  748. SDValue N1 = Op.getOperand(1);
  749. SDValue NN1;
  750. if (N0 == N1)
  751. NN1 = NN0;
  752. else {
  753. NN1 = PromoteOperand(N1, PVT, Replace1);
  754. if (NN1.getNode() == 0)
  755. return SDValue();
  756. }
  757. AddToWorkList(NN0.getNode());
  758. if (NN1.getNode())
  759. AddToWorkList(NN1.getNode());
  760. if (Replace0)
  761. ReplaceLoadWithPromotedLoad(N0.getNode(), NN0.getNode());
  762. if (Replace1)
  763. ReplaceLoadWithPromotedLoad(N1.getNode(), NN1.getNode());
  764. DEBUG(dbgs() << "\nPromoting ";
  765. Op.getNode()->dump(&DAG));
  766. SDLoc dl(Op);
  767. return DAG.getNode(ISD::TRUNCATE, dl, VT,
  768. DAG.getNode(Opc, dl, PVT, NN0, NN1));
  769. }
  770. return SDValue();
  771. }
  772. /// PromoteIntShiftOp - Promote the specified integer shift operation if the
  773. /// target indicates it is beneficial. e.g. On x86, it's usually better to
  774. /// promote i16 operations to i32 since i16 instructions are longer.
  775. SDValue DAGCombiner::PromoteIntShiftOp(SDValue Op) {
  776. if (!LegalOperations)
  777. return SDValue();
  778. EVT VT = Op.getValueType();
  779. if (VT.isVector() || !VT.isInteger())
  780. return SDValue();
  781. // If operation type is 'undesirable', e.g. i16 on x86, consider
  782. // promoting it.
  783. unsigned Opc = Op.getOpcode();
  784. if (TLI.isTypeDesirableForOp(Opc, VT))
  785. return SDValue();
  786. EVT PVT = VT;
  787. // Consult target whether it is a good idea to promote this operation and
  788. // what's the right type to promote it to.
  789. if (TLI.IsDesirableToPromoteOp(Op, PVT)) {
  790. assert(PVT != VT && "Don't know what type to promote to!");
  791. bool Replace = false;
  792. SDValue N0 = Op.getOperand(0);
  793. if (Opc == ISD::SRA)
  794. N0 = SExtPromoteOperand(Op.getOperand(0), PVT);
  795. else if (Opc == ISD::SRL)
  796. N0 = ZExtPromoteOperand(Op.getOperand(0), PVT);
  797. else
  798. N0 = PromoteOperand(N0, PVT, Replace);
  799. if (N0.getNode() == 0)
  800. return SDValue();
  801. AddToWorkList(N0.getNode());
  802. if (Replace)
  803. ReplaceLoadWithPromotedLoad(Op.getOperand(0).getNode(), N0.getNode());
  804. DEBUG(dbgs() << "\nPromoting ";
  805. Op.getNode()->dump(&DAG));
  806. SDLoc dl(Op);
  807. return DAG.getNode(ISD::TRUNCATE, dl, VT,
  808. DAG.getNode(Opc, dl, PVT, N0, Op.getOperand(1)));
  809. }
  810. return SDValue();
  811. }
  812. SDValue DAGCombiner::PromoteExtend(SDValue Op) {
  813. if (!LegalOperations)
  814. return SDValue();
  815. EVT VT = Op.getValueType();
  816. if (VT.isVector() || !VT.isInteger())
  817. return SDValue();
  818. // If operation type is 'undesirable', e.g. i16 on x86, consider
  819. // promoting it.
  820. unsigned Opc = Op.getOpcode();
  821. if (TLI.isTypeDesirableForOp(Opc, VT))
  822. return SDValue();
  823. EVT PVT = VT;
  824. // Consult target whether it is a good idea to promote this operation and
  825. // what's the right type to promote it to.
  826. if (TLI.IsDesirableToPromoteOp(Op, PVT)) {
  827. assert(PVT != VT && "Don't know what type to promote to!");
  828. // fold (aext (aext x)) -> (aext x)
  829. // fold (aext (zext x)) -> (zext x)
  830. // fold (aext (sext x)) -> (sext x)
  831. DEBUG(dbgs() << "\nPromoting ";
  832. Op.getNode()->dump(&DAG));
  833. return DAG.getNode(Op.getOpcode(), SDLoc(Op), VT, Op.getOperand(0));
  834. }
  835. return SDValue();
  836. }
  837. bool DAGCombiner::PromoteLoad(SDValue Op) {
  838. if (!LegalOperations)
  839. return false;
  840. EVT VT = Op.getValueType();
  841. if (VT.isVector() || !VT.isInteger())
  842. return false;
  843. // If operation type is 'undesirable', e.g. i16 on x86, consider
  844. // promoting it.
  845. unsigned Opc = Op.getOpcode();
  846. if (TLI.isTypeDesirableForOp(Opc, VT))
  847. return false;
  848. EVT PVT = VT;
  849. // Consult target whether it is a good idea to promote this operation and
  850. // what's the right type to promote it to.
  851. if (TLI.IsDesirableToPromoteOp(Op, PVT)) {
  852. assert(PVT != VT && "Don't know what type to promote to!");
  853. SDLoc dl(Op);
  854. SDNode *N = Op.getNode();
  855. LoadSDNode *LD = cast<LoadSDNode>(N);
  856. EVT MemVT = LD->getMemoryVT();
  857. ISD::LoadExtType ExtType = ISD::isNON_EXTLoad(LD)
  858. ? (TLI.isLoadExtLegal(ISD::ZEXTLOAD, MemVT) ? ISD::ZEXTLOAD
  859. : ISD::EXTLOAD)
  860. : LD->getExtensionType();
  861. SDValue NewLD = DAG.getExtLoad(ExtType, dl, PVT,
  862. LD->getChain(), LD->getBasePtr(),
  863. MemVT, LD->getMemOperand());
  864. SDValue Result = DAG.getNode(ISD::TRUNCATE, dl, VT, NewLD);
  865. DEBUG(dbgs() << "\nPromoting ";
  866. N->dump(&DAG);
  867. dbgs() << "\nTo: ";
  868. Result.getNode()->dump(&DAG);
  869. dbgs() << '\n');
  870. WorkListRemover DeadNodes(*this);
  871. DAG.ReplaceAllUsesOfValueWith(SDValue(N, 0), Result);
  872. DAG.ReplaceAllUsesOfValueWith(SDValue(N, 1), NewLD.getValue(1));
  873. removeFromWorkList(N);
  874. DAG.DeleteNode(N);
  875. AddToWorkList(Result.getNode());
  876. return true;
  877. }
  878. return false;
  879. }
  880. //===----------------------------------------------------------------------===//
  881. // Main DAG Combiner implementation
  882. //===----------------------------------------------------------------------===//
  883. void DAGCombiner::Run(CombineLevel AtLevel) {
  884. // set the instance variables, so that the various visit routines may use it.
  885. Level = AtLevel;
  886. LegalOperations = Level >= AfterLegalizeVectorOps;
  887. LegalTypes = Level >= AfterLegalizeTypes;
  888. // Add all the dag nodes to the worklist.
  889. for (SelectionDAG::allnodes_iterator I = DAG.allnodes_begin(),
  890. E = DAG.allnodes_end(); I != E; ++I)
  891. AddToWorkList(I);
  892. // Create a dummy node (which is not added to allnodes), that adds a reference
  893. // to the root node, preventing it from being deleted, and tracking any
  894. // changes of the root.
  895. HandleSDNode Dummy(DAG.getRoot());
  896. // The root of the dag may dangle to deleted nodes until the dag combiner is
  897. // done. Set it to null to avoid confusion.
  898. DAG.setRoot(SDValue());
  899. // while the worklist isn't empty, find a node and
  900. // try and combine it.
  901. while (!WorkListContents.empty()) {
  902. SDNode *N;
  903. // The WorkListOrder holds the SDNodes in order, but it may contain
  904. // duplicates.
  905. // In order to avoid a linear scan, we use a set (O(log N)) to hold what the
  906. // worklist *should* contain, and check the node we want to visit is should
  907. // actually be visited.
  908. do {
  909. N = WorkListOrder.pop_back_val();
  910. } while (!WorkListContents.erase(N));
  911. // If N has no uses, it is dead. Make sure to revisit all N's operands once
  912. // N is deleted from the DAG, since they too may now be dead or may have a
  913. // reduced number of uses, allowing other xforms.
  914. if (N->use_empty() && N != &Dummy) {
  915. for (unsigned i = 0, e = N->getNumOperands(); i != e; ++i)
  916. AddToWorkList(N->getOperand(i).getNode());
  917. DAG.DeleteNode(N);
  918. continue;
  919. }
  920. SDValue RV = combine(N);
  921. if (RV.getNode() == 0)
  922. continue;
  923. ++NodesCombined;
  924. // If we get back the same node we passed in, rather than a new node or
  925. // zero, we know that the node must have defined multiple values and
  926. // CombineTo was used. Since CombineTo takes care of the worklist
  927. // mechanics for us, we have no work to do in this case.
  928. if (RV.getNode() == N)
  929. continue;
  930. assert(N->getOpcode() != ISD::DELETED_NODE &&
  931. RV.getNode()->getOpcode() != ISD::DELETED_NODE &&
  932. "Node was deleted but visit returned new node!");
  933. DEBUG(dbgs() << "\nReplacing.3 ";
  934. N->dump(&DAG);
  935. dbgs() << "\nWith: ";
  936. RV.getNode()->dump(&DAG);
  937. dbgs() << '\n');
  938. // Transfer debug value.
  939. DAG.TransferDbgValues(SDValue(N, 0), RV);
  940. WorkListRemover DeadNodes(*this);
  941. if (N->getNumValues() == RV.getNode()->getNumValues())
  942. DAG.ReplaceAllUsesWith(N, RV.getNode());
  943. else {
  944. assert(N->getValueType(0) == RV.getValueType() &&
  945. N->getNumValues() == 1 && "Type mismatch");
  946. SDValue OpV = RV;
  947. DAG.ReplaceAllUsesWith(N, &OpV);
  948. }
  949. // Push the new node and any users onto the worklist
  950. AddToWorkList(RV.getNode());
  951. AddUsersToWorkList(RV.getNode());
  952. // Add any uses of the old node to the worklist in case this node is the
  953. // last one that uses them. They may become dead after this node is
  954. // deleted.
  955. for (unsigned i = 0, e = N->getNumOperands(); i != e; ++i)
  956. AddToWorkList(N->getOperand(i).getNode());
  957. // Finally, if the node is now dead, remove it from the graph. The node
  958. // may not be dead if the replacement process recursively simplified to
  959. // something else needing this node.
  960. if (N->use_empty()) {
  961. // Nodes can be reintroduced into the worklist. Make sure we do not
  962. // process a node that has been replaced.
  963. removeFromWorkList(N);
  964. // Finally, since the node is now dead, remove it from the graph.
  965. DAG.DeleteNode(N);
  966. }
  967. }
  968. // If the root changed (e.g. it was a dead load, update the root).
  969. DAG.setRoot(Dummy.getValue());
  970. DAG.RemoveDeadNodes();
  971. }
  972. SDValue DAGCombiner::visit(SDNode *N) {
  973. switch (N->getOpcode()) {
  974. default: break;
  975. case ISD::TokenFactor: return visitTokenFactor(N);
  976. case ISD::MERGE_VALUES: return visitMERGE_VALUES(N);
  977. case ISD::ADD: return visitADD(N);
  978. case ISD::SUB: return visitSUB(N);
  979. case ISD::ADDC: return visitADDC(N);
  980. case ISD::SUBC: return visitSUBC(N);
  981. case ISD::ADDE: return visitADDE(N);
  982. case ISD::SUBE: return visitSUBE(N);
  983. case ISD::MUL: return visitMUL(N);
  984. case ISD::SDIV: return visitSDIV(N);
  985. case ISD::UDIV: return visitUDIV(N);
  986. case ISD::SREM: return visitSREM(N);
  987. case ISD::UREM: return visitUREM(N);
  988. case ISD::MULHU: return visitMULHU(N);
  989. case ISD::MULHS: return visitMULHS(N);
  990. case ISD::SMUL_LOHI: return visitSMUL_LOHI(N);
  991. case ISD::UMUL_LOHI: return visitUMUL_LOHI(N);
  992. case ISD::SMULO: return visitSMULO(N);
  993. case ISD::UMULO: return visitUMULO(N);
  994. case ISD::SDIVREM: return visitSDIVREM(N);
  995. case ISD::UDIVREM: return visitUDIVREM(N);
  996. case ISD::AND: return visitAND(N);
  997. case ISD::OR: return visitOR(N);
  998. case ISD::XOR: return visitXOR(N);
  999. case ISD::SHL: return visitSHL(N);
  1000. case ISD::SRA: return visitSRA(N);
  1001. case ISD::SRL: return visitSRL(N);
  1002. case ISD::CTLZ: return visitCTLZ(N);
  1003. case ISD::CTLZ_ZERO_UNDEF: return visitCTLZ_ZERO_UNDEF(N);
  1004. case ISD::CTTZ: return visitCTTZ(N);
  1005. case ISD::CTTZ_ZERO_UNDEF: return visitCTTZ_ZERO_UNDEF(N);
  1006. case ISD::CTPOP: return visitCTPOP(N);
  1007. case ISD::SELECT: return visitSELECT(N);
  1008. case ISD::VSELECT: return visitVSELECT(N);
  1009. case ISD::SELECT_CC: return visitSELECT_CC(N);
  1010. case ISD::SETCC: return visitSETCC(N);
  1011. case ISD::SIGN_EXTEND: return visitSIGN_EXTEND(N);
  1012. case ISD::ZERO_EXTEND: return visitZERO_EXTEND(N);
  1013. case ISD::ANY_EXTEND: return visitANY_EXTEND(N);
  1014. case ISD::SIGN_EXTEND_INREG: return visitSIGN_EXTEND_INREG(N);
  1015. case ISD::TRUNCATE: return visitTRUNCATE(N);
  1016. case ISD::BITCAST: return visitBITCAST(N);
  1017. case ISD::BUILD_PAIR: return visitBUILD_PAIR(N);
  1018. case ISD::FADD: return visitFADD(N);
  1019. case ISD::FSUB: return visitFSUB(N);
  1020. case ISD::FMUL: return visitFMUL(N);
  1021. case ISD::FMA: return visitFMA(N);
  1022. case ISD::FDIV: return visitFDIV(N);
  1023. case ISD::FREM: return visitFREM(N);
  1024. case ISD::FCOPYSIGN: return visitFCOPYSIGN(N);
  1025. case ISD::SINT_TO_FP: return visitSINT_TO_FP(N);
  1026. case ISD::UINT_TO_FP: return visitUINT_TO_FP(N);
  1027. case ISD::FP_TO_SINT: return visitFP_TO_SINT(N);
  1028. case ISD::FP_TO_UINT: return visitFP_TO_UINT(N);
  1029. case ISD::FP_ROUND: return visitFP_ROUND(N);
  1030. case ISD::FP_ROUND_INREG: return visitFP_ROUND_INREG(N);
  1031. case ISD::FP_EXTEND: return visitFP_EXTEND(N);
  1032. case ISD::FNEG: return visitFNEG(N);
  1033. case ISD::FABS: return visitFABS(N);
  1034. case ISD::FFLOOR: return visitFFLOOR(N);
  1035. case ISD::FCEIL: return visitFCEIL(N);
  1036. case ISD::FTRUNC: return visitFTRUNC(N);
  1037. case ISD::BRCOND: return visitBRCOND(N);
  1038. case ISD::BR_CC: return visitBR_CC(N);
  1039. case ISD::LOAD: return visitLOAD(N);
  1040. case ISD::STORE: return visitSTORE(N);
  1041. case ISD::INSERT_VECTOR_ELT: return visitINSERT_VECTOR_ELT(N);
  1042. case ISD::EXTRACT_VECTOR_ELT: return visitEXTRACT_VECTOR_ELT(N);
  1043. case ISD::BUILD_VECTOR: return visitBUILD_VECTOR(N);
  1044. case ISD::CONCAT_VECTORS: return visitCONCAT_VECTORS(N);
  1045. case ISD::EXTRACT_SUBVECTOR: return visitEXTRACT_SUBVECTOR(N);
  1046. case ISD::VECTOR_SHUFFLE: return visitVECTOR_SHUFFLE(N);
  1047. }
  1048. return SDValue();
  1049. }
  1050. SDValue DAGCombiner::combine(SDNode *N) {
  1051. SDValue RV = visit(N);
  1052. // If nothing happened, try a target-specific DAG combine.
  1053. if (RV.getNode() == 0) {
  1054. assert(N->getOpcode() != ISD::DELETED_NODE &&
  1055. "Node was deleted but visit returned NULL!");
  1056. if (N->getOpcode() >= ISD::BUILTIN_OP_END ||
  1057. TLI.hasTargetDAGCombine((ISD::NodeType)N->getOpcode())) {
  1058. // Expose the DAG combiner to the target combiner impls.
  1059. TargetLowering::DAGCombinerInfo
  1060. DagCombineInfo(DAG, Level, false, this);
  1061. RV = TLI.PerformDAGCombine(N, DagCombineInfo);
  1062. }
  1063. }
  1064. // If nothing happened still, try promoting the operation.
  1065. if (RV.getNode() == 0) {
  1066. switch (N->getOpcode()) {
  1067. default: break;
  1068. case ISD::ADD:
  1069. case ISD::SUB:
  1070. case ISD::MUL:
  1071. case ISD::AND:
  1072. case ISD::OR:
  1073. case ISD::XOR:
  1074. RV = PromoteIntBinOp(SDValue(N, 0));
  1075. break;
  1076. case ISD::SHL:
  1077. case ISD::SRA:
  1078. case ISD::SRL:
  1079. RV = PromoteIntShiftOp(SDValue(N, 0));
  1080. break;
  1081. case ISD::SIGN_EXTEND:
  1082. case ISD::ZERO_EXTEND:
  1083. case ISD::ANY_EXTEND:
  1084. RV = PromoteExtend(SDValue(N, 0));
  1085. break;
  1086. case ISD::LOAD:
  1087. if (PromoteLoad(SDValue(N, 0)))
  1088. RV = SDValue(N, 0);
  1089. break;
  1090. }
  1091. }
  1092. // If N is a commutative binary node, try commuting it to enable more
  1093. // sdisel CSE.
  1094. if (RV.getNode() == 0 &&
  1095. SelectionDAG::isCommutativeBinOp(N->getOpcode()) &&
  1096. N->getNumValues() == 1) {
  1097. SDValue N0 = N->getOperand(0);
  1098. SDValue N1 = N->getOperand(1);
  1099. // Constant operands are canonicalized to RHS.
  1100. if (isa<ConstantSDNode>(N0) || !isa<ConstantSDNode>(N1)) {
  1101. SDValue Ops[] = { N1, N0 };
  1102. SDNode *CSENode = DAG.getNodeIfExists(N->getOpcode(), N->getVTList(),
  1103. Ops, 2);
  1104. if (CSENode)
  1105. return SDValue(CSENode, 0);
  1106. }
  1107. }
  1108. return RV;
  1109. }
  1110. /// getInputChainForNode - Given a node, return its input chain if it has one,
  1111. /// otherwise return a null sd operand.
  1112. static SDValue getInputChainForNode(SDNode *N) {
  1113. if (unsigned NumOps = N->getNumOperands()) {
  1114. if (N->getOperand(0).getValueType() == MVT::Other)
  1115. return N->getOperand(0);
  1116. if (N->getOperand(NumOps-1).getValueType() == MVT::Other)
  1117. return N->getOperand(NumOps-1);
  1118. for (unsigned i = 1; i < NumOps-1; ++i)
  1119. if (N->getOperand(i).getValueType() == MVT::Other)
  1120. return N->getOperand(i);
  1121. }
  1122. return SDValue();
  1123. }
  1124. SDValue DAGCombiner::visitTokenFactor(SDNode *N) {
  1125. // If N has two operands, where one has an input chain equal to the other,
  1126. // the 'other' chain is redundant.
  1127. if (N->getNumOperands() == 2) {
  1128. if (getInputChainForNode(N->getOperand(0).getNode()) == N->getOperand(1))
  1129. return N->getOperand(0);
  1130. if (getInputChainForNode(N->getOperand(1).getNode()) == N->getOperand(0))
  1131. return N->getOperand(1);
  1132. }
  1133. SmallVector<SDNode *, 8> TFs; // List of token factors to visit.
  1134. SmallVector<SDValue, 8> Ops; // Ops for replacing token factor.
  1135. SmallPtrSet<SDNode*, 16> SeenOps;
  1136. bool Changed = false; // If we should replace this token factor.
  1137. // Start out with this token factor.
  1138. TFs.push_back(N);
  1139. // Iterate through token factors. The TFs grows when new token factors are
  1140. // encountered.
  1141. for (unsigned i = 0; i < TFs.size(); ++i) {
  1142. SDNode *TF = TFs[i];
  1143. // Check each of the operands.
  1144. for (unsigned i = 0, ie = TF->getNumOperands(); i != ie; ++i) {
  1145. SDValue Op = TF->getOperand(i);
  1146. switch (Op.getOpcode()) {
  1147. case ISD::EntryToken:
  1148. // Entry tokens don't need to be added to the list. They are
  1149. // rededundant.
  1150. Changed = true;
  1151. break;
  1152. case ISD::TokenFactor:
  1153. if (Op.hasOneUse() &&
  1154. std::find(TFs.begin(), TFs.end(), Op.getNode()) == TFs.end()) {
  1155. // Q…