/mordor/scheduler.h
C Header | 255 lines | 118 code | 40 blank | 97 comment | 8 complexity | f318e18f2747120f08fde949bd5edb8a MD5 | raw file
1#ifndef __MORDOR_SCHEDULER_H__ 2#define __MORDOR_SCHEDULER_H__ 3// Copyright (c) 2009 - Mozy, Inc. 4 5#include <list> 6 7#include <boost/function.hpp> 8#include <boost/noncopyable.hpp> 9#include <boost/shared_ptr.hpp> 10#include <boost/thread/mutex.hpp> 11 12#include "thread.h" 13#include "thread_local_storage.h" 14 15namespace Mordor { 16 17class Fiber; 18 19/// Cooperative user-mode thread (Fiber) Scheduler 20 21/// A Scheduler is used to cooperatively schedule fibers on threads, 22/// implementing an M-on-N threading model. A Scheduler can either "hijack" 23/// the thread it was created on (by passing useCaller = true in the 24/// constructor), spawn multiple threads of its own, or a hybrid of the two. 25/// 26/// Hijacking and Schedulers begin processing fibers when either 27/// yieldTo() or dispatch() is called. The Scheduler will stop itself when 28/// there are no more Fibers scheduled, and return from yieldTo() or 29/// dispatch(). Hybrid and spawned Schedulers must be explicitly stopped via 30/// stop(). stop() will return only after there are no more Fibers scheduled. 31class Scheduler : public boost::noncopyable 32{ 33public: 34 /// Default constructor 35 36 /// By default, a single-threaded hijacking Scheduler is constructed. 37 /// If threads > 1 && useCaller == true, a hybrid Scheduler is constructed. 38 /// If useCaller == false, this Scheduler will not be associated with 39 /// the currently executing thread. 40 /// @param threads How many threads this Scheduler should be comprised of 41 /// @param useCaller If this Scheduler should "hijack" the currently 42 /// executing thread 43 /// @param batchSize Number of operations to pull off the scheduler queue 44 /// on every iteration 45 /// @pre if (useCaller == true) Scheduler::getThis() == NULL 46 Scheduler(size_t threads = 1, bool useCaller = true, size_t batchSize = 1); 47 /// Destroys the scheduler, implicitly calling stop() 48 virtual ~Scheduler(); 49 50 /// @return The Scheduler controlling the currently executing thread 51 static Scheduler* getThis(); 52 53 /// Explicitly start the Scheduler 54 55 /// Derived classes should call start() in their constructor. 56 /// It is safe to call start() even if the Scheduler is already started - 57 /// it will be a no-op 58 void start(); 59 60 /// Explicitly stop the scheduler 61 62 /// This must be called for hybrid and spawned Schedulers. It is safe to 63 /// call stop() even if the Scheduler is already stopped (or stopping) - 64 /// it will be a no-op 65 /// For hybrid or hijacking schedulers, it must be called from within 66 /// the scheduler. For spawned Schedulers, it must be called from outside 67 /// the Scheduler. 68 /// If called on a hybrid/hijacking scheduler from a Fiber 69 /// that did not create the Scheduler, it will return immediately (the 70 /// Scheduler will yield to the creating Fiber when all work is complete). 71 /// In all other cases stop() will not return until all work is complete. 72 void stop(); 73 74 /// Schedule a Fiber to be executed on the Scheduler 75 76 /// @param fd The Fiber or the functor to schedule, if a pointer is passed 77 /// in, the ownership will be transfered to this scheduler 78 /// @param thread Optionally provide a specific thread for the Fiber to run 79 /// on 80 template <class FiberOrDg> 81 void schedule(FiberOrDg fd, tid_t thread = emptytid()) 82 { 83 bool tickleMe; 84 { 85 boost::mutex::scoped_lock lock(m_mutex); 86 tickleMe = scheduleNoLock(fd, thread); 87 } 88 if (shouldTickle(tickleMe)) 89 tickle(); 90 } 91 92 /// Schedule multiple items to be executed at once 93 94 /// @param begin The first item to schedule 95 /// @param end One past the last item to schedule 96 template <class InputIterator> 97 void schedule(InputIterator begin, InputIterator end) 98 { 99 bool tickleMe = false; 100 { 101 boost::mutex::scoped_lock lock(m_mutex); 102 while (begin != end) { 103 tickleMe = scheduleNoLock(&*begin) || tickleMe; 104 ++begin; 105 } 106 } 107 if (shouldTickle(tickleMe)) 108 tickle(); 109 } 110 111 /// Change the currently executing Fiber to be running on this Scheduler 112 113 /// This function can be used to change which Scheduler/thread the 114 /// currently executing Fiber is executing on. This switch is done by 115 /// rescheduling this Fiber on this Scheduler, and yielding to the current 116 /// Scheduler. 117 /// @param thread Optionally provide a specific thread for this Fiber to 118 /// run on 119 /// @post Scheduler::getThis() == this 120 void switchTo(tid_t thread = emptytid()); 121 122 /// Yield to the Scheduler to allow other Fibers to execute on this thread 123 124 /// The Scheduler will not re-schedule this Fiber automatically. 125 /// 126 /// In a hijacking Scheduler, any scheduled work will begin running if 127 /// someone yields to the Scheduler. 128 /// @pre Scheduler::getThis() != NULL 129 static void yieldTo(); 130 131 /// Yield to the Scheduler to allow other Fibers to execute on this thread 132 133 /// The Scheduler will automatically re-schedule this Fiber. 134 /// @pre Scheduler::getThis() != NULL 135 static void yield(); 136 137 /// Force a hijacking Scheduler to process scheduled work 138 139 /// Calls yieldTo(), and yields back to the currently executing Fiber 140 /// when there is no more work to be done 141 /// @pre this is a hijacking Scheduler 142 void dispatch(); 143 144 size_t threadCount() const 145 { 146 return m_threadCount + (m_rootFiber ? 1 : 0); 147 } 148 /// Change the number of threads in this scheduler 149 void threadCount(size_t threads); 150 151 const std::vector<boost::shared_ptr<Thread> >& threads() const 152 { 153 return m_threads; 154 } 155 156 tid_t rootThreadId() const { return m_rootThread; } 157protected: 158 /// Derived classes can query stopping() to see if the Scheduler is trying 159 /// to stop, and should return from the idle Fiber as soon as possible. 160 /// 161 /// Also, this function should be implemented if the derived class has 162 /// any additional work to do in the idle Fiber that the Scheduler is not 163 /// aware of. 164 virtual bool stopping(); 165 166 /// The function called (in its own Fiber) when there is no work scheduled 167 /// on the Scheduler. The Scheduler is not considered stopped until the 168 /// idle Fiber has terminated. 169 /// 170 /// Implementors should Fiber::yield() when it believes there is work 171 /// scheduled on the Scheduler. 172 virtual void idle() = 0; 173 /// The Scheduler wants to force the idle fiber to Fiber::yield(), because 174 /// new work has been scheduled. 175 virtual void tickle() = 0; 176 177 bool hasWorkToDo(); 178 virtual bool hasIdleThreads() const { return m_idleThreadCount != 0; } 179 180 /// determine whether tickle() is needed, to be invoked in schedule() 181 /// @param empty whether m_fibers is empty before the new task is scheduled 182 virtual bool shouldTickle(bool empty) const 183 { return empty && Scheduler::getThis() != this; } 184 185 /// set `this' to TLS so that getThis() can get correct Scheduler 186 void setThis() { t_scheduler = this; } 187 188private: 189 void yieldTo(bool yieldToCallerOnTerminate); 190 void run(); 191 192 /// @pre @c fd should be valid 193 /// @pre the task to be scheduled is not thread-targeted, or this scheduler 194 /// owns the targeted thread. 195 template <class FiberOrDg> 196 bool scheduleNoLock(FiberOrDg fd, 197 tid_t thread = emptytid()) { 198 bool tickleMe = m_fibers.empty(); 199 m_fibers.push_back(FiberAndThread(fd, thread)); 200 return tickleMe; 201 } 202 203private: 204 struct FiberAndThread { 205 boost::shared_ptr<Fiber> fiber; 206 boost::function<void ()> dg; 207 tid_t thread; 208 FiberAndThread(boost::shared_ptr<Fiber> f, tid_t th) 209 : fiber(f), thread(th) {} 210 FiberAndThread(boost::shared_ptr<Fiber>* f, tid_t th) 211 : thread(th) { 212 fiber.swap(*f); 213 } 214 FiberAndThread(boost::function<void ()> d, tid_t th) 215 : dg(d), thread(th) {} 216 FiberAndThread(boost::function<void ()> *d, tid_t th) 217 : thread(th) { 218 dg.swap(*d); 219 } 220 }; 221 static ThreadLocalStorage<Scheduler *> t_scheduler; 222 static ThreadLocalStorage<Fiber *> t_fiber; 223 boost::mutex m_mutex; 224 std::list<FiberAndThread> m_fibers; 225 tid_t m_rootThread; 226 boost::shared_ptr<Fiber> m_rootFiber; 227 boost::shared_ptr<Fiber> m_callingFiber; 228 std::vector<boost::shared_ptr<Thread> > m_threads; 229 size_t m_threadCount, m_activeThreadCount, m_idleThreadCount; 230 bool m_stopping; 231 bool m_autoStop; 232 size_t m_batchSize; 233}; 234 235/// Automatic Scheduler switcher 236 237/// Automatically returns to Scheduler::getThis() when goes out of scope 238/// (by calling Scheduler::switchTo()) 239struct SchedulerSwitcher : public boost::noncopyable 240{ 241public: 242 /// Captures Scheduler::getThis(), and optionally calls target->switchTo() 243 /// if target != NULL 244 SchedulerSwitcher(Scheduler *target = NULL); 245 /// Calls switchTo() on the Scheduler captured in the constructor 246 /// @post Scheduler::getThis() == the Scheduler captured in the constructor 247 ~SchedulerSwitcher(); 248 249private: 250 Scheduler *m_caller; 251}; 252 253} 254 255#endif