PageRenderTime 92ms CodeModel.GetById 17ms app.highlight 63ms RepoModel.GetById 0ms app.codeStats 1ms

/3rd_party/llvm/lib/CodeGen/RegAllocLinearScan.cpp

https://code.google.com/p/softart/
C++ | 1543 lines | 1064 code | 197 blank | 282 comment | 308 complexity | d4371db70422068c05f461884bceb37d MD5 | raw file
Possible License(s): LGPL-2.1, BSD-3-Clause, JSON, MPL-2.0-no-copyleft-exception, GPL-2.0, GPL-3.0, LGPL-3.0, BSD-2-Clause

Large files files are truncated, but you can click here to view the full file

   1//===-- RegAllocLinearScan.cpp - Linear Scan register allocator -----------===//
   2//
   3//                     The LLVM Compiler Infrastructure
   4//
   5// This file is distributed under the University of Illinois Open Source
   6// License. See LICENSE.TXT for details.
   7//
   8//===----------------------------------------------------------------------===//
   9//
  10// This file implements a linear scan register allocator.
  11//
  12//===----------------------------------------------------------------------===//
  13
  14#define DEBUG_TYPE "regalloc"
  15#include "LiveDebugVariables.h"
  16#include "LiveRangeEdit.h"
  17#include "VirtRegMap.h"
  18#include "VirtRegRewriter.h"
  19#include "RegisterClassInfo.h"
  20#include "Spiller.h"
  21#include "llvm/Analysis/AliasAnalysis.h"
  22#include "llvm/Function.h"
  23#include "llvm/CodeGen/CalcSpillWeights.h"
  24#include "llvm/CodeGen/LiveIntervalAnalysis.h"
  25#include "llvm/CodeGen/MachineFunctionPass.h"
  26#include "llvm/CodeGen/MachineInstr.h"
  27#include "llvm/CodeGen/MachineLoopInfo.h"
  28#include "llvm/CodeGen/MachineRegisterInfo.h"
  29#include "llvm/CodeGen/Passes.h"
  30#include "llvm/CodeGen/RegAllocRegistry.h"
  31#include "llvm/Target/TargetRegisterInfo.h"
  32#include "llvm/Target/TargetMachine.h"
  33#include "llvm/Target/TargetOptions.h"
  34#include "llvm/Target/TargetInstrInfo.h"
  35#include "llvm/ADT/EquivalenceClasses.h"
  36#include "llvm/ADT/SmallSet.h"
  37#include "llvm/ADT/Statistic.h"
  38#include "llvm/ADT/STLExtras.h"
  39#include "llvm/Support/Debug.h"
  40#include "llvm/Support/ErrorHandling.h"
  41#include "llvm/Support/raw_ostream.h"
  42#include <algorithm>
  43#include <queue>
  44#include <memory>
  45#include <cmath>
  46
  47using namespace llvm;
  48
  49STATISTIC(NumIters     , "Number of iterations performed");
  50STATISTIC(NumBacktracks, "Number of times we had to backtrack");
  51STATISTIC(NumCoalesce,   "Number of copies coalesced");
  52STATISTIC(NumDowngrade,  "Number of registers downgraded");
  53
  54static cl::opt<bool>
  55NewHeuristic("new-spilling-heuristic",
  56             cl::desc("Use new spilling heuristic"),
  57             cl::init(false), cl::Hidden);
  58
  59static cl::opt<bool>
  60TrivCoalesceEnds("trivial-coalesce-ends",
  61                  cl::desc("Attempt trivial coalescing of interval ends"),
  62                  cl::init(false), cl::Hidden);
  63
  64static cl::opt<bool>
  65AvoidWAWHazard("avoid-waw-hazard",
  66               cl::desc("Avoid write-write hazards for some register classes"),
  67               cl::init(false), cl::Hidden);
  68
  69static RegisterRegAlloc
  70linearscanRegAlloc("linearscan", "linear scan register allocator",
  71                   createLinearScanRegisterAllocator);
  72
  73namespace {
  74  // When we allocate a register, add it to a fixed-size queue of
  75  // registers to skip in subsequent allocations. This trades a small
  76  // amount of register pressure and increased spills for flexibility in
  77  // the post-pass scheduler.
  78  //
  79  // Note that in a the number of registers used for reloading spills
  80  // will be one greater than the value of this option.
  81  //
  82  // One big limitation of this is that it doesn't differentiate between
  83  // different register classes. So on x86-64, if there is xmm register
  84  // pressure, it can caused fewer GPRs to be held in the queue.
  85  static cl::opt<unsigned>
  86  NumRecentlyUsedRegs("linearscan-skip-count",
  87                      cl::desc("Number of registers for linearscan to remember"
  88                               "to skip."),
  89                      cl::init(0),
  90                      cl::Hidden);
  91
  92  struct RALinScan : public MachineFunctionPass {
  93    static char ID;
  94    RALinScan() : MachineFunctionPass(ID) {
  95      initializeLiveDebugVariablesPass(*PassRegistry::getPassRegistry());
  96      initializeLiveIntervalsPass(*PassRegistry::getPassRegistry());
  97      initializeStrongPHIEliminationPass(*PassRegistry::getPassRegistry());
  98      initializeRegisterCoalescerPass(
  99        *PassRegistry::getPassRegistry());
 100      initializeCalculateSpillWeightsPass(*PassRegistry::getPassRegistry());
 101      initializeLiveStacksPass(*PassRegistry::getPassRegistry());
 102      initializeMachineDominatorTreePass(*PassRegistry::getPassRegistry());
 103      initializeMachineLoopInfoPass(*PassRegistry::getPassRegistry());
 104      initializeVirtRegMapPass(*PassRegistry::getPassRegistry());
 105      initializeMachineDominatorTreePass(*PassRegistry::getPassRegistry());
 106      
 107      // Initialize the queue to record recently-used registers.
 108      if (NumRecentlyUsedRegs > 0)
 109        RecentRegs.resize(NumRecentlyUsedRegs, 0);
 110      RecentNext = RecentRegs.begin();
 111      avoidWAW_ = 0;
 112    }
 113
 114    typedef std::pair<LiveInterval*, LiveInterval::iterator> IntervalPtr;
 115    typedef SmallVector<IntervalPtr, 32> IntervalPtrs;
 116  private:
 117    /// RelatedRegClasses - This structure is built the first time a function is
 118    /// compiled, and keeps track of which register classes have registers that
 119    /// belong to multiple classes or have aliases that are in other classes.
 120    EquivalenceClasses<const TargetRegisterClass*> RelatedRegClasses;
 121    DenseMap<unsigned, const TargetRegisterClass*> OneClassForEachPhysReg;
 122
 123    // NextReloadMap - For each register in the map, it maps to the another
 124    // register which is defined by a reload from the same stack slot and
 125    // both reloads are in the same basic block.
 126    DenseMap<unsigned, unsigned> NextReloadMap;
 127
 128    // DowngradedRegs - A set of registers which are being "downgraded", i.e.
 129    // un-favored for allocation.
 130    SmallSet<unsigned, 8> DowngradedRegs;
 131
 132    // DowngradeMap - A map from virtual registers to physical registers being
 133    // downgraded for the virtual registers.
 134    DenseMap<unsigned, unsigned> DowngradeMap;
 135
 136    MachineFunction* mf_;
 137    MachineRegisterInfo* mri_;
 138    const TargetMachine* tm_;
 139    const TargetRegisterInfo* tri_;
 140    const TargetInstrInfo* tii_;
 141    BitVector allocatableRegs_;
 142    BitVector reservedRegs_;
 143    LiveIntervals* li_;
 144    MachineLoopInfo *loopInfo;
 145    RegisterClassInfo RegClassInfo;
 146
 147    /// handled_ - Intervals are added to the handled_ set in the order of their
 148    /// start value.  This is uses for backtracking.
 149    std::vector<LiveInterval*> handled_;
 150
 151    /// fixed_ - Intervals that correspond to machine registers.
 152    ///
 153    IntervalPtrs fixed_;
 154
 155    /// active_ - Intervals that are currently being processed, and which have a
 156    /// live range active for the current point.
 157    IntervalPtrs active_;
 158
 159    /// inactive_ - Intervals that are currently being processed, but which have
 160    /// a hold at the current point.
 161    IntervalPtrs inactive_;
 162
 163    typedef std::priority_queue<LiveInterval*,
 164                                SmallVector<LiveInterval*, 64>,
 165                                greater_ptr<LiveInterval> > IntervalHeap;
 166    IntervalHeap unhandled_;
 167
 168    /// regUse_ - Tracks register usage.
 169    SmallVector<unsigned, 32> regUse_;
 170    SmallVector<unsigned, 32> regUseBackUp_;
 171
 172    /// vrm_ - Tracks register assignments.
 173    VirtRegMap* vrm_;
 174
 175    std::auto_ptr<VirtRegRewriter> rewriter_;
 176
 177    std::auto_ptr<Spiller> spiller_;
 178
 179    // The queue of recently-used registers.
 180    SmallVector<unsigned, 4> RecentRegs;
 181    SmallVector<unsigned, 4>::iterator RecentNext;
 182
 183    // Last write-after-write register written.
 184    unsigned avoidWAW_;
 185
 186    // Record that we just picked this register.
 187    void recordRecentlyUsed(unsigned reg) {
 188      assert(reg != 0 && "Recently used register is NOREG!");
 189      if (!RecentRegs.empty()) {
 190        *RecentNext++ = reg;
 191        if (RecentNext == RecentRegs.end())
 192          RecentNext = RecentRegs.begin();
 193      }
 194    }
 195
 196  public:
 197    virtual const char* getPassName() const {
 198      return "Linear Scan Register Allocator";
 199    }
 200
 201    virtual void getAnalysisUsage(AnalysisUsage &AU) const {
 202      AU.setPreservesCFG();
 203      AU.addRequired<AliasAnalysis>();
 204      AU.addPreserved<AliasAnalysis>();
 205      AU.addRequired<LiveIntervals>();
 206      AU.addPreserved<SlotIndexes>();
 207      if (StrongPHIElim)
 208        AU.addRequiredID(StrongPHIEliminationID);
 209      // Make sure PassManager knows which analyses to make available
 210      // to coalescing and which analyses coalescing invalidates.
 211      AU.addRequiredTransitiveID(RegisterCoalescerPassID);
 212      AU.addRequired<CalculateSpillWeights>();
 213      AU.addRequiredID(LiveStacksID);
 214      AU.addPreservedID(LiveStacksID);
 215      AU.addRequired<MachineLoopInfo>();
 216      AU.addPreserved<MachineLoopInfo>();
 217      AU.addRequired<VirtRegMap>();
 218      AU.addPreserved<VirtRegMap>();
 219      AU.addRequired<LiveDebugVariables>();
 220      AU.addPreserved<LiveDebugVariables>();
 221      AU.addRequiredID(MachineDominatorsID);
 222      AU.addPreservedID(MachineDominatorsID);
 223      MachineFunctionPass::getAnalysisUsage(AU);
 224    }
 225
 226    /// runOnMachineFunction - register allocate the whole function
 227    bool runOnMachineFunction(MachineFunction&);
 228
 229    // Determine if we skip this register due to its being recently used.
 230    bool isRecentlyUsed(unsigned reg) const {
 231      return reg == avoidWAW_ ||
 232       std::find(RecentRegs.begin(), RecentRegs.end(), reg) != RecentRegs.end();
 233    }
 234
 235  private:
 236    /// linearScan - the linear scan algorithm
 237    void linearScan();
 238
 239    /// initIntervalSets - initialize the interval sets.
 240    ///
 241    void initIntervalSets();
 242
 243    /// processActiveIntervals - expire old intervals and move non-overlapping
 244    /// ones to the inactive list.
 245    void processActiveIntervals(SlotIndex CurPoint);
 246
 247    /// processInactiveIntervals - expire old intervals and move overlapping
 248    /// ones to the active list.
 249    void processInactiveIntervals(SlotIndex CurPoint);
 250
 251    /// hasNextReloadInterval - Return the next liveinterval that's being
 252    /// defined by a reload from the same SS as the specified one.
 253    LiveInterval *hasNextReloadInterval(LiveInterval *cur);
 254
 255    /// DowngradeRegister - Downgrade a register for allocation.
 256    void DowngradeRegister(LiveInterval *li, unsigned Reg);
 257
 258    /// UpgradeRegister - Upgrade a register for allocation.
 259    void UpgradeRegister(unsigned Reg);
 260
 261    /// assignRegOrStackSlotAtInterval - assign a register if one
 262    /// is available, or spill.
 263    void assignRegOrStackSlotAtInterval(LiveInterval* cur);
 264
 265    void updateSpillWeights(std::vector<float> &Weights,
 266                            unsigned reg, float weight,
 267                            const TargetRegisterClass *RC);
 268
 269    /// findIntervalsToSpill - Determine the intervals to spill for the
 270    /// specified interval. It's passed the physical registers whose spill
 271    /// weight is the lowest among all the registers whose live intervals
 272    /// conflict with the interval.
 273    void findIntervalsToSpill(LiveInterval *cur,
 274                            std::vector<std::pair<unsigned,float> > &Candidates,
 275                            unsigned NumCands,
 276                            SmallVector<LiveInterval*, 8> &SpillIntervals);
 277
 278    /// attemptTrivialCoalescing - If a simple interval is defined by a copy,
 279    /// try to allocate the definition to the same register as the source,
 280    /// if the register is not defined during the life time of the interval.
 281    /// This eliminates a copy, and is used to coalesce copies which were not
 282    /// coalesced away before allocation either due to dest and src being in
 283    /// different register classes or because the coalescer was overly
 284    /// conservative.
 285    unsigned attemptTrivialCoalescing(LiveInterval &cur, unsigned Reg);
 286
 287    ///
 288    /// Register usage / availability tracking helpers.
 289    ///
 290
 291    void initRegUses() {
 292      regUse_.resize(tri_->getNumRegs(), 0);
 293      regUseBackUp_.resize(tri_->getNumRegs(), 0);
 294    }
 295
 296    void finalizeRegUses() {
 297#ifndef NDEBUG
 298      // Verify all the registers are "freed".
 299      bool Error = false;
 300      for (unsigned i = 0, e = tri_->getNumRegs(); i != e; ++i) {
 301        if (regUse_[i] != 0) {
 302          dbgs() << tri_->getName(i) << " is still in use!\n";
 303          Error = true;
 304        }
 305      }
 306      if (Error)
 307        llvm_unreachable(0);
 308#endif
 309      regUse_.clear();
 310      regUseBackUp_.clear();
 311    }
 312
 313    void addRegUse(unsigned physReg) {
 314      assert(TargetRegisterInfo::isPhysicalRegister(physReg) &&
 315             "should be physical register!");
 316      ++regUse_[physReg];
 317      for (const unsigned* as = tri_->getAliasSet(physReg); *as; ++as)
 318        ++regUse_[*as];
 319    }
 320
 321    void delRegUse(unsigned physReg) {
 322      assert(TargetRegisterInfo::isPhysicalRegister(physReg) &&
 323             "should be physical register!");
 324      assert(regUse_[physReg] != 0);
 325      --regUse_[physReg];
 326      for (const unsigned* as = tri_->getAliasSet(physReg); *as; ++as) {
 327        assert(regUse_[*as] != 0);
 328        --regUse_[*as];
 329      }
 330    }
 331
 332    bool isRegAvail(unsigned physReg) const {
 333      assert(TargetRegisterInfo::isPhysicalRegister(physReg) &&
 334             "should be physical register!");
 335      return regUse_[physReg] == 0;
 336    }
 337
 338    void backUpRegUses() {
 339      regUseBackUp_ = regUse_;
 340    }
 341
 342    void restoreRegUses() {
 343      regUse_ = regUseBackUp_;
 344    }
 345
 346    ///
 347    /// Register handling helpers.
 348    ///
 349
 350    /// getFreePhysReg - return a free physical register for this virtual
 351    /// register interval if we have one, otherwise return 0.
 352    unsigned getFreePhysReg(LiveInterval* cur);
 353    unsigned getFreePhysReg(LiveInterval* cur,
 354                            const TargetRegisterClass *RC,
 355                            unsigned MaxInactiveCount,
 356                            SmallVector<unsigned, 256> &inactiveCounts,
 357                            bool SkipDGRegs);
 358
 359    /// getFirstNonReservedPhysReg - return the first non-reserved physical
 360    /// register in the register class.
 361    unsigned getFirstNonReservedPhysReg(const TargetRegisterClass *RC) {
 362      ArrayRef<unsigned> O = RegClassInfo.getOrder(RC);
 363      assert(!O.empty() && "All registers reserved?!");
 364      return O.front();
 365    }
 366
 367    void ComputeRelatedRegClasses();
 368
 369    template <typename ItTy>
 370    void printIntervals(const char* const str, ItTy i, ItTy e) const {
 371      DEBUG({
 372          if (str)
 373            dbgs() << str << " intervals:\n";
 374
 375          for (; i != e; ++i) {
 376            dbgs() << '\t' << *i->first << " -> ";
 377
 378            unsigned reg = i->first->reg;
 379            if (TargetRegisterInfo::isVirtualRegister(reg))
 380              reg = vrm_->getPhys(reg);
 381
 382            dbgs() << tri_->getName(reg) << '\n';
 383          }
 384        });
 385    }
 386  };
 387  char RALinScan::ID = 0;
 388}
 389
 390INITIALIZE_PASS_BEGIN(RALinScan, "linearscan-regalloc",
 391                      "Linear Scan Register Allocator", false, false)
 392INITIALIZE_PASS_DEPENDENCY(LiveIntervals)
 393INITIALIZE_PASS_DEPENDENCY(StrongPHIElimination)
 394INITIALIZE_PASS_DEPENDENCY(CalculateSpillWeights)
 395INITIALIZE_PASS_DEPENDENCY(LiveStacks)
 396INITIALIZE_PASS_DEPENDENCY(MachineLoopInfo)
 397INITIALIZE_PASS_DEPENDENCY(VirtRegMap)
 398INITIALIZE_PASS_DEPENDENCY(RegisterCoalescer)
 399INITIALIZE_AG_DEPENDENCY(AliasAnalysis)
 400INITIALIZE_PASS_END(RALinScan, "linearscan-regalloc",
 401                    "Linear Scan Register Allocator", false, false)
 402
 403void RALinScan::ComputeRelatedRegClasses() {
 404  // First pass, add all reg classes to the union, and determine at least one
 405  // reg class that each register is in.
 406  bool HasAliases = false;
 407  for (TargetRegisterInfo::regclass_iterator RCI = tri_->regclass_begin(),
 408       E = tri_->regclass_end(); RCI != E; ++RCI) {
 409    RelatedRegClasses.insert(*RCI);
 410    for (TargetRegisterClass::iterator I = (*RCI)->begin(), E = (*RCI)->end();
 411         I != E; ++I) {
 412      HasAliases = HasAliases || *tri_->getAliasSet(*I) != 0;
 413
 414      const TargetRegisterClass *&PRC = OneClassForEachPhysReg[*I];
 415      if (PRC) {
 416        // Already processed this register.  Just make sure we know that
 417        // multiple register classes share a register.
 418        RelatedRegClasses.unionSets(PRC, *RCI);
 419      } else {
 420        PRC = *RCI;
 421      }
 422    }
 423  }
 424
 425  // Second pass, now that we know conservatively what register classes each reg
 426  // belongs to, add info about aliases.  We don't need to do this for targets
 427  // without register aliases.
 428  if (HasAliases)
 429    for (DenseMap<unsigned, const TargetRegisterClass*>::iterator
 430         I = OneClassForEachPhysReg.begin(), E = OneClassForEachPhysReg.end();
 431         I != E; ++I)
 432      for (const unsigned *AS = tri_->getAliasSet(I->first); *AS; ++AS) {
 433        const TargetRegisterClass *AliasClass = 
 434          OneClassForEachPhysReg.lookup(*AS);
 435        if (AliasClass)
 436          RelatedRegClasses.unionSets(I->second, AliasClass);
 437      }
 438}
 439
 440/// attemptTrivialCoalescing - If a simple interval is defined by a copy, try
 441/// allocate the definition the same register as the source register if the
 442/// register is not defined during live time of the interval. If the interval is
 443/// killed by a copy, try to use the destination register. This eliminates a
 444/// copy. This is used to coalesce copies which were not coalesced away before
 445/// allocation either due to dest and src being in different register classes or
 446/// because the coalescer was overly conservative.
 447unsigned RALinScan::attemptTrivialCoalescing(LiveInterval &cur, unsigned Reg) {
 448  unsigned Preference = vrm_->getRegAllocPref(cur.reg);
 449  if ((Preference && Preference == Reg) || !cur.containsOneValue())
 450    return Reg;
 451
 452  // We cannot handle complicated live ranges. Simple linear stuff only.
 453  if (cur.ranges.size() != 1)
 454    return Reg;
 455
 456  const LiveRange &range = cur.ranges.front();
 457
 458  VNInfo *vni = range.valno;
 459  if (vni->isUnused() || !vni->def.isValid())
 460    return Reg;
 461
 462  unsigned CandReg;
 463  {
 464    MachineInstr *CopyMI;
 465    if ((CopyMI = li_->getInstructionFromIndex(vni->def)) && CopyMI->isCopy())
 466      // Defined by a copy, try to extend SrcReg forward
 467      CandReg = CopyMI->getOperand(1).getReg();
 468    else if (TrivCoalesceEnds &&
 469            (CopyMI = li_->getInstructionFromIndex(range.end.getBaseIndex())) &&
 470             CopyMI->isCopy() && cur.reg == CopyMI->getOperand(1).getReg())
 471      // Only used by a copy, try to extend DstReg backwards
 472      CandReg = CopyMI->getOperand(0).getReg();
 473    else
 474      return Reg;
 475
 476    // If the target of the copy is a sub-register then don't coalesce.
 477    if(CopyMI->getOperand(0).getSubReg())
 478      return Reg;
 479  }
 480
 481  if (TargetRegisterInfo::isVirtualRegister(CandReg)) {
 482    if (!vrm_->isAssignedReg(CandReg))
 483      return Reg;
 484    CandReg = vrm_->getPhys(CandReg);
 485  }
 486  if (Reg == CandReg)
 487    return Reg;
 488
 489  const TargetRegisterClass *RC = mri_->getRegClass(cur.reg);
 490  if (!RC->contains(CandReg))
 491    return Reg;
 492
 493  if (li_->conflictsWithPhysReg(cur, *vrm_, CandReg))
 494    return Reg;
 495
 496  // Try to coalesce.
 497  DEBUG(dbgs() << "Coalescing: " << cur << " -> " << tri_->getName(CandReg)
 498        << '\n');
 499  vrm_->clearVirt(cur.reg);
 500  vrm_->assignVirt2Phys(cur.reg, CandReg);
 501
 502  ++NumCoalesce;
 503  return CandReg;
 504}
 505
 506bool RALinScan::runOnMachineFunction(MachineFunction &fn) {
 507  mf_ = &fn;
 508  mri_ = &fn.getRegInfo();
 509  tm_ = &fn.getTarget();
 510  tri_ = tm_->getRegisterInfo();
 511  tii_ = tm_->getInstrInfo();
 512  allocatableRegs_ = tri_->getAllocatableSet(fn);
 513  reservedRegs_ = tri_->getReservedRegs(fn);
 514  li_ = &getAnalysis<LiveIntervals>();
 515  loopInfo = &getAnalysis<MachineLoopInfo>();
 516  RegClassInfo.runOnMachineFunction(fn);
 517
 518  // We don't run the coalescer here because we have no reason to
 519  // interact with it.  If the coalescer requires interaction, it
 520  // won't do anything.  If it doesn't require interaction, we assume
 521  // it was run as a separate pass.
 522
 523  // If this is the first function compiled, compute the related reg classes.
 524  if (RelatedRegClasses.empty())
 525    ComputeRelatedRegClasses();
 526
 527  // Also resize register usage trackers.
 528  initRegUses();
 529
 530  vrm_ = &getAnalysis<VirtRegMap>();
 531  if (!rewriter_.get()) rewriter_.reset(createVirtRegRewriter());
 532
 533  spiller_.reset(createSpiller(*this, *mf_, *vrm_));
 534
 535  initIntervalSets();
 536
 537  linearScan();
 538
 539  // Rewrite spill code and update the PhysRegsUsed set.
 540  rewriter_->runOnMachineFunction(*mf_, *vrm_, li_);
 541
 542  // Write out new DBG_VALUE instructions.
 543  getAnalysis<LiveDebugVariables>().emitDebugValues(vrm_);
 544
 545  assert(unhandled_.empty() && "Unhandled live intervals remain!");
 546
 547  finalizeRegUses();
 548
 549  fixed_.clear();
 550  active_.clear();
 551  inactive_.clear();
 552  handled_.clear();
 553  NextReloadMap.clear();
 554  DowngradedRegs.clear();
 555  DowngradeMap.clear();
 556  spiller_.reset(0);
 557
 558  return true;
 559}
 560
 561/// initIntervalSets - initialize the interval sets.
 562///
 563void RALinScan::initIntervalSets()
 564{
 565  assert(unhandled_.empty() && fixed_.empty() &&
 566         active_.empty() && inactive_.empty() &&
 567         "interval sets should be empty on initialization");
 568
 569  handled_.reserve(li_->getNumIntervals());
 570
 571  for (LiveIntervals::iterator i = li_->begin(), e = li_->end(); i != e; ++i) {
 572    if (TargetRegisterInfo::isPhysicalRegister(i->second->reg)) {
 573      if (!i->second->empty() && allocatableRegs_.test(i->second->reg)) {
 574        mri_->setPhysRegUsed(i->second->reg);
 575        fixed_.push_back(std::make_pair(i->second, i->second->begin()));
 576      }
 577    } else {
 578      if (i->second->empty()) {
 579        assignRegOrStackSlotAtInterval(i->second);
 580      }
 581      else
 582        unhandled_.push(i->second);
 583    }
 584  }
 585}
 586
 587void RALinScan::linearScan() {
 588  // linear scan algorithm
 589  DEBUG({
 590      dbgs() << "********** LINEAR SCAN **********\n"
 591             << "********** Function: "
 592             << mf_->getFunction()->getName() << '\n';
 593      printIntervals("fixed", fixed_.begin(), fixed_.end());
 594    });
 595
 596  while (!unhandled_.empty()) {
 597    // pick the interval with the earliest start point
 598    LiveInterval* cur = unhandled_.top();
 599    unhandled_.pop();
 600    ++NumIters;
 601    DEBUG(dbgs() << "\n*** CURRENT ***: " << *cur << '\n');
 602
 603    assert(!cur->empty() && "Empty interval in unhandled set.");
 604
 605    processActiveIntervals(cur->beginIndex());
 606    processInactiveIntervals(cur->beginIndex());
 607
 608    assert(TargetRegisterInfo::isVirtualRegister(cur->reg) &&
 609           "Can only allocate virtual registers!");
 610
 611    // Allocating a virtual register. try to find a free
 612    // physical register or spill an interval (possibly this one) in order to
 613    // assign it one.
 614    assignRegOrStackSlotAtInterval(cur);
 615
 616    DEBUG({
 617        printIntervals("active", active_.begin(), active_.end());
 618        printIntervals("inactive", inactive_.begin(), inactive_.end());
 619      });
 620  }
 621
 622  // Expire any remaining active intervals
 623  while (!active_.empty()) {
 624    IntervalPtr &IP = active_.back();
 625    unsigned reg = IP.first->reg;
 626    DEBUG(dbgs() << "\tinterval " << *IP.first << " expired\n");
 627    assert(TargetRegisterInfo::isVirtualRegister(reg) &&
 628           "Can only allocate virtual registers!");
 629    reg = vrm_->getPhys(reg);
 630    delRegUse(reg);
 631    active_.pop_back();
 632  }
 633
 634  // Expire any remaining inactive intervals
 635  DEBUG({
 636      for (IntervalPtrs::reverse_iterator
 637             i = inactive_.rbegin(); i != inactive_.rend(); ++i)
 638        dbgs() << "\tinterval " << *i->first << " expired\n";
 639    });
 640  inactive_.clear();
 641
 642  // Add live-ins to every BB except for entry. Also perform trivial coalescing.
 643  MachineFunction::iterator EntryMBB = mf_->begin();
 644  SmallVector<MachineBasicBlock*, 8> LiveInMBBs;
 645  for (LiveIntervals::iterator i = li_->begin(), e = li_->end(); i != e; ++i) {
 646    LiveInterval &cur = *i->second;
 647    unsigned Reg = 0;
 648    bool isPhys = TargetRegisterInfo::isPhysicalRegister(cur.reg);
 649    if (isPhys)
 650      Reg = cur.reg;
 651    else if (vrm_->isAssignedReg(cur.reg))
 652      Reg = attemptTrivialCoalescing(cur, vrm_->getPhys(cur.reg));
 653    if (!Reg)
 654      continue;
 655    // Ignore splited live intervals.
 656    if (!isPhys && vrm_->getPreSplitReg(cur.reg))
 657      continue;
 658
 659    for (LiveInterval::Ranges::const_iterator I = cur.begin(), E = cur.end();
 660         I != E; ++I) {
 661      const LiveRange &LR = *I;
 662      if (li_->findLiveInMBBs(LR.start, LR.end, LiveInMBBs)) {
 663        for (unsigned i = 0, e = LiveInMBBs.size(); i != e; ++i)
 664          if (LiveInMBBs[i] != EntryMBB) {
 665            assert(TargetRegisterInfo::isPhysicalRegister(Reg) &&
 666                   "Adding a virtual register to livein set?");
 667            LiveInMBBs[i]->addLiveIn(Reg);
 668          }
 669        LiveInMBBs.clear();
 670      }
 671    }
 672  }
 673
 674  DEBUG(dbgs() << *vrm_);
 675
 676  // Look for physical registers that end up not being allocated even though
 677  // register allocator had to spill other registers in its register class.
 678  if (!vrm_->FindUnusedRegisters(li_))
 679    return;
 680}
 681
 682/// processActiveIntervals - expire old intervals and move non-overlapping ones
 683/// to the inactive list.
 684void RALinScan::processActiveIntervals(SlotIndex CurPoint)
 685{
 686  DEBUG(dbgs() << "\tprocessing active intervals:\n");
 687
 688  for (unsigned i = 0, e = active_.size(); i != e; ++i) {
 689    LiveInterval *Interval = active_[i].first;
 690    LiveInterval::iterator IntervalPos = active_[i].second;
 691    unsigned reg = Interval->reg;
 692
 693    IntervalPos = Interval->advanceTo(IntervalPos, CurPoint);
 694
 695    if (IntervalPos == Interval->end()) {     // Remove expired intervals.
 696      DEBUG(dbgs() << "\t\tinterval " << *Interval << " expired\n");
 697      assert(TargetRegisterInfo::isVirtualRegister(reg) &&
 698             "Can only allocate virtual registers!");
 699      reg = vrm_->getPhys(reg);
 700      delRegUse(reg);
 701
 702      // Pop off the end of the list.
 703      active_[i] = active_.back();
 704      active_.pop_back();
 705      --i; --e;
 706
 707    } else if (IntervalPos->start > CurPoint) {
 708      // Move inactive intervals to inactive list.
 709      DEBUG(dbgs() << "\t\tinterval " << *Interval << " inactive\n");
 710      assert(TargetRegisterInfo::isVirtualRegister(reg) &&
 711             "Can only allocate virtual registers!");
 712      reg = vrm_->getPhys(reg);
 713      delRegUse(reg);
 714      // add to inactive.
 715      inactive_.push_back(std::make_pair(Interval, IntervalPos));
 716
 717      // Pop off the end of the list.
 718      active_[i] = active_.back();
 719      active_.pop_back();
 720      --i; --e;
 721    } else {
 722      // Otherwise, just update the iterator position.
 723      active_[i].second = IntervalPos;
 724    }
 725  }
 726}
 727
 728/// processInactiveIntervals - expire old intervals and move overlapping
 729/// ones to the active list.
 730void RALinScan::processInactiveIntervals(SlotIndex CurPoint)
 731{
 732  DEBUG(dbgs() << "\tprocessing inactive intervals:\n");
 733
 734  for (unsigned i = 0, e = inactive_.size(); i != e; ++i) {
 735    LiveInterval *Interval = inactive_[i].first;
 736    LiveInterval::iterator IntervalPos = inactive_[i].second;
 737    unsigned reg = Interval->reg;
 738
 739    IntervalPos = Interval->advanceTo(IntervalPos, CurPoint);
 740
 741    if (IntervalPos == Interval->end()) {       // remove expired intervals.
 742      DEBUG(dbgs() << "\t\tinterval " << *Interval << " expired\n");
 743
 744      // Pop off the end of the list.
 745      inactive_[i] = inactive_.back();
 746      inactive_.pop_back();
 747      --i; --e;
 748    } else if (IntervalPos->start <= CurPoint) {
 749      // move re-activated intervals in active list
 750      DEBUG(dbgs() << "\t\tinterval " << *Interval << " active\n");
 751      assert(TargetRegisterInfo::isVirtualRegister(reg) &&
 752             "Can only allocate virtual registers!");
 753      reg = vrm_->getPhys(reg);
 754      addRegUse(reg);
 755      // add to active
 756      active_.push_back(std::make_pair(Interval, IntervalPos));
 757
 758      // Pop off the end of the list.
 759      inactive_[i] = inactive_.back();
 760      inactive_.pop_back();
 761      --i; --e;
 762    } else {
 763      // Otherwise, just update the iterator position.
 764      inactive_[i].second = IntervalPos;
 765    }
 766  }
 767}
 768
 769/// updateSpillWeights - updates the spill weights of the specifed physical
 770/// register and its weight.
 771void RALinScan::updateSpillWeights(std::vector<float> &Weights,
 772                                   unsigned reg, float weight,
 773                                   const TargetRegisterClass *RC) {
 774  SmallSet<unsigned, 4> Processed;
 775  SmallSet<unsigned, 4> SuperAdded;
 776  SmallVector<unsigned, 4> Supers;
 777  Weights[reg] += weight;
 778  Processed.insert(reg);
 779  for (const unsigned* as = tri_->getAliasSet(reg); *as; ++as) {
 780    Weights[*as] += weight;
 781    Processed.insert(*as);
 782    if (tri_->isSubRegister(*as, reg) &&
 783        SuperAdded.insert(*as) &&
 784        RC->contains(*as)) {
 785      Supers.push_back(*as);
 786    }
 787  }
 788
 789  // If the alias is a super-register, and the super-register is in the
 790  // register class we are trying to allocate. Then add the weight to all
 791  // sub-registers of the super-register even if they are not aliases.
 792  // e.g. allocating for GR32, bh is not used, updating bl spill weight.
 793  //      bl should get the same spill weight otherwise it will be chosen
 794  //      as a spill candidate since spilling bh doesn't make ebx available.
 795  for (unsigned i = 0, e = Supers.size(); i != e; ++i) {
 796    for (const unsigned *sr = tri_->getSubRegisters(Supers[i]); *sr; ++sr)
 797      if (!Processed.count(*sr))
 798        Weights[*sr] += weight;
 799  }
 800}
 801
 802static
 803RALinScan::IntervalPtrs::iterator
 804FindIntervalInVector(RALinScan::IntervalPtrs &IP, LiveInterval *LI) {
 805  for (RALinScan::IntervalPtrs::iterator I = IP.begin(), E = IP.end();
 806       I != E; ++I)
 807    if (I->first == LI) return I;
 808  return IP.end();
 809}
 810
 811static void RevertVectorIteratorsTo(RALinScan::IntervalPtrs &V,
 812                                    SlotIndex Point){
 813  for (unsigned i = 0, e = V.size(); i != e; ++i) {
 814    RALinScan::IntervalPtr &IP = V[i];
 815    LiveInterval::iterator I = std::upper_bound(IP.first->begin(),
 816                                                IP.second, Point);
 817    if (I != IP.first->begin()) --I;
 818    IP.second = I;
 819  }
 820}
 821
 822/// getConflictWeight - Return the number of conflicts between cur
 823/// live interval and defs and uses of Reg weighted by loop depthes.
 824static
 825float getConflictWeight(LiveInterval *cur, unsigned Reg, LiveIntervals *li_,
 826                        MachineRegisterInfo *mri_,
 827                        MachineLoopInfo *loopInfo) {
 828  float Conflicts = 0;
 829  for (MachineRegisterInfo::reg_iterator I = mri_->reg_begin(Reg),
 830         E = mri_->reg_end(); I != E; ++I) {
 831    MachineInstr *MI = &*I;
 832    if (cur->liveAt(li_->getInstructionIndex(MI))) {
 833      unsigned loopDepth = loopInfo->getLoopDepth(MI->getParent());
 834      Conflicts += std::pow(10.0f, (float)loopDepth);
 835    }
 836  }
 837  return Conflicts;
 838}
 839
 840/// findIntervalsToSpill - Determine the intervals to spill for the
 841/// specified interval. It's passed the physical registers whose spill
 842/// weight is the lowest among all the registers whose live intervals
 843/// conflict with the interval.
 844void RALinScan::findIntervalsToSpill(LiveInterval *cur,
 845                            std::vector<std::pair<unsigned,float> > &Candidates,
 846                            unsigned NumCands,
 847                            SmallVector<LiveInterval*, 8> &SpillIntervals) {
 848  // We have figured out the *best* register to spill. But there are other
 849  // registers that are pretty good as well (spill weight within 3%). Spill
 850  // the one that has fewest defs and uses that conflict with cur.
 851  float Conflicts[3] = { 0.0f, 0.0f, 0.0f };
 852  SmallVector<LiveInterval*, 8> SLIs[3];
 853
 854  DEBUG({
 855      dbgs() << "\tConsidering " << NumCands << " candidates: ";
 856      for (unsigned i = 0; i != NumCands; ++i)
 857        dbgs() << tri_->getName(Candidates[i].first) << " ";
 858      dbgs() << "\n";
 859    });
 860
 861  // Calculate the number of conflicts of each candidate.
 862  for (IntervalPtrs::iterator i = active_.begin(); i != active_.end(); ++i) {
 863    unsigned Reg = i->first->reg;
 864    unsigned PhysReg = vrm_->getPhys(Reg);
 865    if (!cur->overlapsFrom(*i->first, i->second))
 866      continue;
 867    for (unsigned j = 0; j < NumCands; ++j) {
 868      unsigned Candidate = Candidates[j].first;
 869      if (tri_->regsOverlap(PhysReg, Candidate)) {
 870        if (NumCands > 1)
 871          Conflicts[j] += getConflictWeight(cur, Reg, li_, mri_, loopInfo);
 872        SLIs[j].push_back(i->first);
 873      }
 874    }
 875  }
 876
 877  for (IntervalPtrs::iterator i = inactive_.begin(); i != inactive_.end(); ++i){
 878    unsigned Reg = i->first->reg;
 879    unsigned PhysReg = vrm_->getPhys(Reg);
 880    if (!cur->overlapsFrom(*i->first, i->second-1))
 881      continue;
 882    for (unsigned j = 0; j < NumCands; ++j) {
 883      unsigned Candidate = Candidates[j].first;
 884      if (tri_->regsOverlap(PhysReg, Candidate)) {
 885        if (NumCands > 1)
 886          Conflicts[j] += getConflictWeight(cur, Reg, li_, mri_, loopInfo);
 887        SLIs[j].push_back(i->first);
 888      }
 889    }
 890  }
 891
 892  // Which is the best candidate?
 893  unsigned BestCandidate = 0;
 894  float MinConflicts = Conflicts[0];
 895  for (unsigned i = 1; i != NumCands; ++i) {
 896    if (Conflicts[i] < MinConflicts) {
 897      BestCandidate = i;
 898      MinConflicts = Conflicts[i];
 899    }
 900  }
 901
 902  std::copy(SLIs[BestCandidate].begin(), SLIs[BestCandidate].end(),
 903            std::back_inserter(SpillIntervals));
 904}
 905
 906namespace {
 907  struct WeightCompare {
 908  private:
 909    const RALinScan &Allocator;
 910
 911  public:
 912    WeightCompare(const RALinScan &Alloc) : Allocator(Alloc) {}
 913
 914    typedef std::pair<unsigned, float> RegWeightPair;
 915    bool operator()(const RegWeightPair &LHS, const RegWeightPair &RHS) const {
 916      return LHS.second < RHS.second && !Allocator.isRecentlyUsed(LHS.first);
 917    }
 918  };
 919}
 920
 921static bool weightsAreClose(float w1, float w2) {
 922  if (!NewHeuristic)
 923    return false;
 924
 925  float diff = w1 - w2;
 926  if (diff <= 0.02f)  // Within 0.02f
 927    return true;
 928  return (diff / w2) <= 0.05f;  // Within 5%.
 929}
 930
 931LiveInterval *RALinScan::hasNextReloadInterval(LiveInterval *cur) {
 932  DenseMap<unsigned, unsigned>::iterator I = NextReloadMap.find(cur->reg);
 933  if (I == NextReloadMap.end())
 934    return 0;
 935  return &li_->getInterval(I->second);
 936}
 937
 938void RALinScan::DowngradeRegister(LiveInterval *li, unsigned Reg) {
 939  for (const unsigned *AS = tri_->getOverlaps(Reg); *AS; ++AS) {
 940    bool isNew = DowngradedRegs.insert(*AS);
 941    (void)isNew; // Silence compiler warning.
 942    assert(isNew && "Multiple reloads holding the same register?");
 943    DowngradeMap.insert(std::make_pair(li->reg, *AS));
 944  }
 945  ++NumDowngrade;
 946}
 947
 948void RALinScan::UpgradeRegister(unsigned Reg) {
 949  if (Reg) {
 950    DowngradedRegs.erase(Reg);
 951    for (const unsigned *AS = tri_->getAliasSet(Reg); *AS; ++AS)
 952      DowngradedRegs.erase(*AS);
 953  }
 954}
 955
 956namespace {
 957  struct LISorter {
 958    bool operator()(LiveInterval* A, LiveInterval* B) {
 959      return A->beginIndex() < B->beginIndex();
 960    }
 961  };
 962}
 963
 964/// assignRegOrStackSlotAtInterval - assign a register if one is available, or
 965/// spill.
 966void RALinScan::assignRegOrStackSlotAtInterval(LiveInterval* cur) {
 967  const TargetRegisterClass *RC = mri_->getRegClass(cur->reg);
 968  DEBUG(dbgs() << "\tallocating current interval from "
 969               << RC->getName() << ": ");
 970
 971  // This is an implicitly defined live interval, just assign any register.
 972  if (cur->empty()) {
 973    unsigned physReg = vrm_->getRegAllocPref(cur->reg);
 974    if (!physReg)
 975      physReg = getFirstNonReservedPhysReg(RC);
 976    DEBUG(dbgs() <<  tri_->getName(physReg) << '\n');
 977    // Note the register is not really in use.
 978    vrm_->assignVirt2Phys(cur->reg, physReg);
 979    return;
 980  }
 981
 982  backUpRegUses();
 983
 984  std::vector<std::pair<unsigned, float> > SpillWeightsToAdd;
 985  SlotIndex StartPosition = cur->beginIndex();
 986  const TargetRegisterClass *RCLeader = RelatedRegClasses.getLeaderValue(RC);
 987
 988  // If start of this live interval is defined by a move instruction and its
 989  // source is assigned a physical register that is compatible with the target
 990  // register class, then we should try to assign it the same register.
 991  // This can happen when the move is from a larger register class to a smaller
 992  // one, e.g. X86::mov32to32_. These move instructions are not coalescable.
 993  if (!vrm_->getRegAllocPref(cur->reg) && cur->hasAtLeastOneValue()) {
 994    VNInfo *vni = cur->begin()->valno;
 995    if (!vni->isUnused() && vni->def.isValid()) {
 996      MachineInstr *CopyMI = li_->getInstructionFromIndex(vni->def);
 997      if (CopyMI && CopyMI->isCopy()) {
 998        unsigned DstSubReg = CopyMI->getOperand(0).getSubReg();
 999        unsigned SrcReg = CopyMI->getOperand(1).getReg();
1000        unsigned SrcSubReg = CopyMI->getOperand(1).getSubReg();
1001        unsigned Reg = 0;
1002        if (TargetRegisterInfo::isPhysicalRegister(SrcReg))
1003          Reg = SrcReg;
1004        else if (vrm_->isAssignedReg(SrcReg))
1005          Reg = vrm_->getPhys(SrcReg);
1006        if (Reg) {
1007          if (SrcSubReg)
1008            Reg = tri_->getSubReg(Reg, SrcSubReg);
1009          if (DstSubReg)
1010            Reg = tri_->getMatchingSuperReg(Reg, DstSubReg, RC);
1011          if (Reg && allocatableRegs_[Reg] && RC->contains(Reg))
1012            mri_->setRegAllocationHint(cur->reg, 0, Reg);
1013        }
1014      }
1015    }
1016  }
1017
1018  // For every interval in inactive we overlap with, mark the
1019  // register as not free and update spill weights.
1020  for (IntervalPtrs::const_iterator i = inactive_.begin(),
1021         e = inactive_.end(); i != e; ++i) {
1022    unsigned Reg = i->first->reg;
1023    assert(TargetRegisterInfo::isVirtualRegister(Reg) &&
1024           "Can only allocate virtual registers!");
1025    const TargetRegisterClass *RegRC = mri_->getRegClass(Reg);
1026    // If this is not in a related reg class to the register we're allocating,
1027    // don't check it.
1028    if (RelatedRegClasses.getLeaderValue(RegRC) == RCLeader &&
1029        cur->overlapsFrom(*i->first, i->second-1)) {
1030      Reg = vrm_->getPhys(Reg);
1031      addRegUse(Reg);
1032      SpillWeightsToAdd.push_back(std::make_pair(Reg, i->first->weight));
1033    }
1034  }
1035
1036  // Speculatively check to see if we can get a register right now.  If not,
1037  // we know we won't be able to by adding more constraints.  If so, we can
1038  // check to see if it is valid.  Doing an exhaustive search of the fixed_ list
1039  // is very bad (it contains all callee clobbered registers for any functions
1040  // with a call), so we want to avoid doing that if possible.
1041  unsigned physReg = getFreePhysReg(cur);
1042  unsigned BestPhysReg = physReg;
1043  if (physReg) {
1044    // We got a register.  However, if it's in the fixed_ list, we might
1045    // conflict with it.  Check to see if we conflict with it or any of its
1046    // aliases.
1047    SmallSet<unsigned, 8> RegAliases;
1048    for (const unsigned *AS = tri_->getAliasSet(physReg); *AS; ++AS)
1049      RegAliases.insert(*AS);
1050
1051    bool ConflictsWithFixed = false;
1052    for (unsigned i = 0, e = fixed_.size(); i != e; ++i) {
1053      IntervalPtr &IP = fixed_[i];
1054      if (physReg == IP.first->reg || RegAliases.count(IP.first->reg)) {
1055        // Okay, this reg is on the fixed list.  Check to see if we actually
1056        // conflict.
1057        LiveInterval *I = IP.first;
1058        if (I->endIndex() > StartPosition) {
1059          LiveInterval::iterator II = I->advanceTo(IP.second, StartPosition);
1060          IP.second = II;
1061          if (II != I->begin() && II->start > StartPosition)
1062            --II;
1063          if (cur->overlapsFrom(*I, II)) {
1064            ConflictsWithFixed = true;
1065            break;
1066          }
1067        }
1068      }
1069    }
1070
1071    // Okay, the register picked by our speculative getFreePhysReg call turned
1072    // out to be in use.  Actually add all of the conflicting fixed registers to
1073    // regUse_ so we can do an accurate query.
1074    if (ConflictsWithFixed) {
1075      // For every interval in fixed we overlap with, mark the register as not
1076      // free and update spill weights.
1077      for (unsigned i = 0, e = fixed_.size(); i != e; ++i) {
1078        IntervalPtr &IP = fixed_[i];
1079        LiveInterval *I = IP.first;
1080
1081        const TargetRegisterClass *RegRC = OneClassForEachPhysReg[I->reg];
1082        if (RelatedRegClasses.getLeaderValue(RegRC) == RCLeader &&
1083            I->endIndex() > StartPosition) {
1084          LiveInterval::iterator II = I->advanceTo(IP.second, StartPosition);
1085          IP.second = II;
1086          if (II != I->begin() && II->start > StartPosition)
1087            --II;
1088          if (cur->overlapsFrom(*I, II)) {
1089            unsigned reg = I->reg;
1090            addRegUse(reg);
1091            SpillWeightsToAdd.push_back(std::make_pair(reg, I->weight));
1092          }
1093        }
1094      }
1095
1096      // Using the newly updated regUse_ object, which includes conflicts in the
1097      // future, see if there are any registers available.
1098      physReg = getFreePhysReg(cur);
1099    }
1100  }
1101
1102  // Restore the physical register tracker, removing information about the
1103  // future.
1104  restoreRegUses();
1105
1106  // If we find a free register, we are done: assign this virtual to
1107  // the free physical register and add this interval to the active
1108  // list.
1109  if (physReg) {
1110    DEBUG(dbgs() <<  tri_->getName(physReg) << '\n');
1111    assert(RC->contains(physReg) && "Invalid candidate");
1112    vrm_->assignVirt2Phys(cur->reg, physReg);
1113    addRegUse(physReg);
1114    active_.push_back(std::make_pair(cur, cur->begin()));
1115    handled_.push_back(cur);
1116
1117    // Remember physReg for avoiding a write-after-write hazard in the next
1118    // instruction.
1119    if (AvoidWAWHazard &&
1120        tri_->avoidWriteAfterWrite(mri_->getRegClass(cur->reg)))
1121      avoidWAW_ = physReg;
1122
1123    // "Upgrade" the physical register since it has been allocated.
1124    UpgradeRegister(physReg);
1125    if (LiveInterval *NextReloadLI = hasNextReloadInterval(cur)) {
1126      // "Downgrade" physReg to try to keep physReg from being allocated until
1127      // the next reload from the same SS is allocated.
1128      mri_->setRegAllocationHint(NextReloadLI->reg, 0, physReg);
1129      DowngradeRegister(cur, physReg);
1130    }
1131    return;
1132  }
1133  DEBUG(dbgs() << "no free registers\n");
1134
1135  // Compile the spill weights into an array that is better for scanning.
1136  std::vector<float> SpillWeights(tri_->getNumRegs(), 0.0f);
1137  for (std::vector<std::pair<unsigned, float> >::iterator
1138       I = SpillWeightsToAdd.begin(), E = SpillWeightsToAdd.end(); I != E; ++I)
1139    updateSpillWeights(SpillWeights, I->first, I->second, RC);
1140
1141  // for each interval in active, update spill weights.
1142  for (IntervalPtrs::const_iterator i = active_.begin(), e = active_.end();
1143       i != e; ++i) {
1144    unsigned reg = i->first->reg;
1145    assert(TargetRegisterInfo::isVirtualRegister(reg) &&
1146           "Can only allocate virtual registers!");
1147    reg = vrm_->getPhys(reg);
1148    updateSpillWeights(SpillWeights, reg, i->first->weight, RC);
1149  }
1150
1151  DEBUG(dbgs() << "\tassigning stack slot at interval "<< *cur << ":\n");
1152
1153  // Find a register to spill.
1154  float minWeight = HUGE_VALF;
1155  unsigned minReg = 0;
1156
1157  bool Found = false;
1158  std::vector<std::pair<unsigned,float> > RegsWeights;
1159  ArrayRef<unsigned> Order = RegClassInfo.getOrder(RC);
1160  if (!minReg || SpillWeights[minReg] == HUGE_VALF)
1161    for (unsigned i = 0; i != Order.size(); ++i) {
1162      unsigned reg = Order[i];
1163      float regWeight = SpillWeights[reg];
1164      // Skip recently allocated registers and reserved registers.
1165      if (minWeight > regWeight && !isRecentlyUsed(reg))
1166        Found = true;
1167      RegsWeights.push_back(std::make_pair(reg, regWeight));
1168    }
1169
1170  // If we didn't find a register that is spillable, try aliases?
1171  if (!Found) {
1172    for (unsigned i = 0; i != Order.size(); ++i) {
1173      unsigned reg = Order[i];
1174      // No need to worry about if the alias register size < regsize of RC.
1175      // We are going to spill all registers that alias it anyway.
1176      for (const unsigned* as = tri_->getAliasSet(reg); *as; ++as)
1177        RegsWeights.push_back(std::make_pair(*as, SpillWeights[*as]));
1178    }
1179  }
1180
1181  // Sort all potential spill candidates by weight.
1182  std::sort(RegsWeights.begin(), RegsWeights.end(), WeightCompare(*this));
1183  minReg = RegsWeights[0].first;
1184  minWeight = RegsWeights[0].second;
1185  if (minWeight == HUGE_VALF) {
1186    // All registers must have inf weight. Just grab one!
1187    minReg = BestPhysReg ? BestPhysReg : getFirstNonReservedPhysReg(RC);
1188    if (cur->weight == HUGE_VALF ||
1189        li_->getApproximateInstructionCount(*cur) == 0) {
1190      // Spill a physical register around defs and uses.
1191      if (li_->spillPhysRegAroundRegDefsUses(*cur, minReg, *vrm_)) {
1192        // spillPhysRegAroundRegDefsUses may have invalidated iterator stored
1193        // in fixed_. Reset them.
1194        for (unsigned i = 0, e = fixed_.size(); i != e; ++i) {
1195          IntervalPtr &IP = fixed_[i];
1196          LiveInterval *I = IP.first;
1197          if (I->reg == minReg || tri_->isSubRegister(minReg, I->reg))
1198            IP.second = I->advanceTo(I->begin(), StartPosition);
1199        }
1200
1201        DowngradedRegs.clear();
1202        assignRegOrStackSlotAtInterval(cur);
1203      } else {
1204        assert(false && "Ran out of registers during register allocation!");
1205        report_fatal_error("Ran out of registers during register allocation!");
1206      }
1207      return;
1208    }
1209  }
1210
1211  // Find up to 3 registers to consider as spill candidates.
1212  unsigned LastCandidate = RegsWeights.size() >= 3 ? 3 : 1;
1213  while (LastCandidate > 1) {
1214    if (weightsAreClose(RegsWeights[LastCandidate-1].second, minWeight))
1215      break;
1216    --LastCandidate;
1217  }
1218
1219  DEBUG({
1220      dbgs() << "\t\tregister(s) with min weight(s): ";
1221
1222      for (unsigned i = 0; i != LastCandidate; ++i)
1223        dbgs() << tri_->getName(RegsWeights[i].first)
1224               << " (" << RegsWeights[i].second << ")\n";
1225    });
1226
1227  // If the current has the minimum weight, we need to spill it and
1228  // add any added intervals back to unhandled, and restart
1229  // linearscan.
1230  if (cur->weight != HUGE_VALF && cur->weight <= minWeight) {
1231    DEBUG(dbgs() << "\t\t\tspilling(c): " << *cur << '\n');
1232    SmallVector<LiveInterval*, 8> added;
1233    LiveRangeEdit LRE(*cur, added);
1234    spiller_->spill(LRE);
1235
1236    std::sort(added.begin(), added.end(), LISorter());
1237    if (added.empty())
1238      return;  // Early exit if all spills were folded.
1239
1240    // Merge added with unhandled.  Note that we have already sorted
1241    // intervals returned by addIntervalsForSpills by their starting
1242    // point.
1243    // This also update the NextReloadMap. That is, it adds mapping from a
1244    // register defined by a reload from SS to the next reload from SS in the
1245    // same basic block.
1246    MachineBasicBlock *LastReloadMBB = 0;
1247    LiveInterval *LastReload = 0;
1248    int LastReloadSS = VirtRegMap::NO_STACK_SLOT;
1249    for (unsigned i = 0, e = added.size(); i != e; ++i) {
1250      LiveInterval *ReloadLi = added[i];
1251      if (ReloadLi->weight == HUGE_VALF &&
1252          li_->getApproximateInstructionCount(*ReloadLi) == 0) {
1253        SlotIndex ReloadIdx = ReloadLi->beginIndex();
1254        MachineBasicBlock *ReloadMBB = li_->getMBBFromIndex(ReloadIdx);
1255        int ReloadSS = vrm_->getStackSlot(ReloadLi->reg);
1256        if (LastReloadMBB == ReloadMBB && LastReloadSS == ReloadSS) {
1257          // Last reload of same SS is in the same MBB. We want to try to
1258          // allocate both reloads the same register and make sure the reg
1259          // isn't clobbered in between if at all possible.
1260          assert(LastReload->beginIndex() < ReloadIdx);
1261          NextReloadMap.insert(std::make_pair(LastReload->reg, ReloadLi->reg));
1262        }
1263        LastReloadMBB = ReloadMBB;
1264        LastReload = ReloadLi;
1265        LastReloadSS = ReloadSS;
1266      }
1267      unhandled_.push(ReloadLi);
1268    }
1269    return;
1270  }
1271
1272  ++NumBacktracks;
1273
1274  // Push the current interval back to unhandled since we are going
1275  // to re-run at least this iteration. Since we didn't modify it it
1276  // should go back right in the front of the list
1277  unhandled_.push(cur);
1278
1279  assert(TargetRegisterInfo::isPhysicalRegister(minReg) &&
1280         "did not choose a register to spill?");
1281
1282  // We spill all intervals aliasing the register with
1283  // minimum weight, rollback to the interval with the earliest
1284  // start point and let the linear scan algorithm run again
1285  SmallVector<LiveInterval*, 8> spillIs;
1286
1287  // Determine which intervals have to be spilled.
1288  findIntervalsToSpill(cur, RegsWeights, LastCandidate, spillIs);
1289
1290  // Set of spilled vregs (used later to rollback properly)
1291  SmallSet<unsigned, 8> spilled;
1292
1293  // The earliest start of a Spilled interval indicates up to where
1294  // in handled we need to roll back
1295  assert(!spillIs.empty() && "No spill intervals?");
1296  SlotIndex earliestStart = spillIs[0]->beginIndex();
1297
1298  // Spill live intervals of virtual regs mapped to the physical register we
1299  // want to clear (and its aliases).  We only spill those that overlap with the
1300  // current interval as the rest do not affect its allocation. we also keep
1301  // track of the earliest start of all spilled live intervals since this will
1302  // mark our rollback point.
1303  SmallVector<LiveInterval*, 8> added;
1304  while (!spillIs.empty()) {
1305    LiveInterval *sli = spillIs.back();
1306    spillIs.pop_back();
1307    DEBUG(dbgs() << "\t\t\tspilling(a): " << *sli << '\n');
1308    if (sli->beginIndex() < earliestStart)
1309      earliestStart = sli->beginIndex();
1310    LiveRangeEdit LRE(*sli, added, 0, &spillIs);
1311    spiller_->spill(LRE);
1312    spilled.insert(sli->reg);
1313  }
1314
1315  // Include any added intervals in earliestStart.
1316  for (unsigned i = 0, e = added.size(); i != e; ++i) {
1317    SlotIndex SI = added[i]->beginIndex();
1318    if (SI < earliestStart)
1319      earliestStart = SI;
1320  }
1321
1322  DEBUG(dbgs() << "\t\trolling back to: " << earliestStart << '\n');
1323
1324

Large files files are truncated, but you can click here to view the full file