/mordor/scheduler.h

  1#ifndef __MORDOR_SCHEDULER_H__
  2#define __MORDOR_SCHEDULER_H__
  3// Copyright (c) 2009 - Mozy, Inc.
  4
  5#include <list>
  6
  7#include <boost/function.hpp>
  8#include <boost/noncopyable.hpp>
  9#include <boost/shared_ptr.hpp>
 10#include <boost/thread/mutex.hpp>
 11
 12#include "thread.h"
 13#include "thread_local_storage.h"
 14
 15namespace Mordor {
 16
 17class Fiber;
 18
 19/// Cooperative user-mode thread (Fiber) Scheduler
 20
 21/// A Scheduler is used to cooperatively schedule fibers on threads,
 22/// implementing an M-on-N threading model. A Scheduler can either "hijack"
 23/// the thread it was created on (by passing useCaller = true in the
 24/// constructor), spawn multiple threads of its own, or a hybrid of the two.
 25///
 26/// Hijacking and Schedulers begin processing fibers when either
 27/// yieldTo() or dispatch() is called. The Scheduler will stop itself when
 28/// there are no more Fibers scheduled, and return from yieldTo() or
 29/// dispatch(). Hybrid and spawned Schedulers must be explicitly stopped via
 30/// stop(). stop() will return only after there are no more Fibers scheduled.
 31class Scheduler : public boost::noncopyable
 32{
 33public:
 34    /// Default constructor
 35
 36    /// By default, a single-threaded hijacking Scheduler is constructed.
 37    /// If threads > 1 && useCaller == true, a hybrid Scheduler is constructed.
 38    /// If useCaller == false, this Scheduler will not be associated with
 39    /// the currently executing thread.
 40    /// @param threads How many threads this Scheduler should be comprised of
 41    /// @param useCaller If this Scheduler should "hijack" the currently
 42    /// executing thread
 43    /// @param batchSize Number of operations to pull off the scheduler queue
 44    /// on every iteration
 45    /// @pre if (useCaller == true) Scheduler::getThis() == NULL
 46    Scheduler(size_t threads = 1, bool useCaller = true, size_t batchSize = 1);
 47    /// Destroys the scheduler, implicitly calling stop()
 48    virtual ~Scheduler();
 49
 50    /// @return The Scheduler controlling the currently executing thread
 51    static Scheduler* getThis();
 52
 53    /// Explicitly start the Scheduler
 54
 55    /// Derived classes should call start() in their constructor.
 56    /// It is safe to call start() even if the Scheduler is already started -
 57    /// it will be a no-op
 58    void start();
 59
 60    /// Explicitly stop the scheduler
 61
 62    /// This must be called for hybrid and spawned Schedulers.  It is safe to
 63    /// call stop() even if the Scheduler is already stopped (or stopping) -
 64    /// it will be a no-op
 65    /// For hybrid or hijacking schedulers, it must be called from within
 66    /// the scheduler.  For spawned Schedulers, it must be called from outside
 67    /// the Scheduler.
 68    /// If called on a hybrid/hijacking scheduler from a Fiber
 69    /// that did not create the Scheduler, it will return immediately (the
 70    /// Scheduler will yield to the creating Fiber when all work is complete).
 71    /// In all other cases stop() will not return until all work is complete.
 72    void stop();
 73
 74    /// Schedule a Fiber to be executed on the Scheduler
 75
 76    /// @param fd The Fiber or the functor to schedule, if a pointer is passed
 77    ///           in, the ownership will be transfered to this scheduler
 78    /// @param thread Optionally provide a specific thread for the Fiber to run
 79    /// on
 80    template <class FiberOrDg>
 81    void schedule(FiberOrDg fd, tid_t thread = emptytid())
 82    {
 83        bool tickleMe;
 84        {
 85            boost::mutex::scoped_lock lock(m_mutex);
 86            tickleMe = scheduleNoLock(fd, thread);
 87        }
 88        if (shouldTickle(tickleMe))
 89            tickle();
 90    }
 91
 92    /// Schedule multiple items to be executed at once
 93
 94    /// @param begin The first item to schedule
 95    /// @param end One past the last item to schedule
 96    template <class InputIterator>
 97    void schedule(InputIterator begin, InputIterator end)
 98    {
 99        bool tickleMe = false;
100        {
101            boost::mutex::scoped_lock lock(m_mutex);
102            while (begin != end) {
103                tickleMe = scheduleNoLock(&*begin) || tickleMe;
104                ++begin;
105            }
106        }
107        if (shouldTickle(tickleMe))
108            tickle();
109    }
110
111    /// Change the currently executing Fiber to be running on this Scheduler
112
113    /// This function can be used to change which Scheduler/thread the
114    /// currently executing Fiber is executing on.  This switch is done by
115    /// rescheduling this Fiber on this Scheduler, and yielding to the current
116    /// Scheduler.
117    /// @param thread Optionally provide a specific thread for this Fiber to
118    /// run on
119    /// @post Scheduler::getThis() == this
120    void switchTo(tid_t thread = emptytid());
121
122    /// Yield to the Scheduler to allow other Fibers to execute on this thread
123
124    /// The Scheduler will not re-schedule this Fiber automatically.
125    ///
126    /// In a hijacking Scheduler, any scheduled work will begin running if
127    /// someone yields to the Scheduler.
128    /// @pre Scheduler::getThis() != NULL
129    static void yieldTo();
130
131    /// Yield to the Scheduler to allow other Fibers to execute on this thread
132
133    /// The Scheduler will automatically re-schedule this Fiber.
134    /// @pre Scheduler::getThis() != NULL
135    static void yield();
136
137    /// Force a hijacking Scheduler to process scheduled work
138
139    /// Calls yieldTo(), and yields back to the currently executing Fiber
140    /// when there is no more work to be done
141    /// @pre this is a hijacking Scheduler
142    void dispatch();
143
144    size_t threadCount() const
145    {
146        return m_threadCount + (m_rootFiber ? 1 : 0);
147    }
148    /// Change the number of threads in this scheduler
149    void threadCount(size_t threads);
150
151    const std::vector<boost::shared_ptr<Thread> >& threads() const
152    {
153        return m_threads;
154    }
155
156    tid_t rootThreadId() const { return m_rootThread; }
157protected:
158    /// Derived classes can query stopping() to see if the Scheduler is trying
159    /// to stop, and should return from the idle Fiber as soon as possible.
160    ///
161    /// Also, this function should be implemented if the derived class has
162    /// any additional work to do in the idle Fiber that the Scheduler is not
163    /// aware of.
164    virtual bool stopping();
165
166    /// The function called (in its own Fiber) when there is no work scheduled
167    /// on the Scheduler.  The Scheduler is not considered stopped until the
168    /// idle Fiber has terminated.
169    ///
170    /// Implementors should Fiber::yield() when it believes there is work
171    /// scheduled on the Scheduler.
172    virtual void idle() = 0;
173    /// The Scheduler wants to force the idle fiber to Fiber::yield(), because
174    /// new work has been scheduled.
175    virtual void tickle() = 0;
176
177    bool hasWorkToDo();
178    virtual bool hasIdleThreads() const { return m_idleThreadCount != 0; }
179
180    /// determine whether tickle() is needed, to be invoked in schedule()
181    /// @param empty whether m_fibers is empty before the new task is scheduled
182    virtual bool shouldTickle(bool empty) const
183    { return empty && Scheduler::getThis() != this; }
184
185    /// set `this' to TLS so that getThis() can get correct Scheduler
186    void setThis() { t_scheduler = this; }
187
188private:
189    void yieldTo(bool yieldToCallerOnTerminate);
190    void run();
191
192    /// @pre @c fd should be valid
193    /// @pre the task to be scheduled is not thread-targeted, or this scheduler
194    ///      owns the targeted thread.
195    template <class FiberOrDg>
196        bool scheduleNoLock(FiberOrDg fd,
197                            tid_t thread = emptytid()) {
198        bool tickleMe = m_fibers.empty();
199        m_fibers.push_back(FiberAndThread(fd, thread));
200        return tickleMe;
201    }
202
203private:
204    struct FiberAndThread {
205        boost::shared_ptr<Fiber> fiber;
206        boost::function<void ()> dg;
207        tid_t thread;
208        FiberAndThread(boost::shared_ptr<Fiber> f, tid_t th)
209            : fiber(f), thread(th) {}
210        FiberAndThread(boost::shared_ptr<Fiber>* f, tid_t th)
211            : thread(th) {
212            fiber.swap(*f);
213        }
214        FiberAndThread(boost::function<void ()> d, tid_t th)
215            : dg(d), thread(th) {}
216        FiberAndThread(boost::function<void ()> *d, tid_t th)
217            : thread(th) {
218            dg.swap(*d);
219        }
220    };
221    static ThreadLocalStorage<Scheduler *> t_scheduler;
222    static ThreadLocalStorage<Fiber *> t_fiber;
223    boost::mutex m_mutex;
224    std::list<FiberAndThread> m_fibers;
225    tid_t m_rootThread;
226    boost::shared_ptr<Fiber> m_rootFiber;
227    boost::shared_ptr<Fiber> m_callingFiber;
228    std::vector<boost::shared_ptr<Thread> > m_threads;
229    size_t m_threadCount, m_activeThreadCount, m_idleThreadCount;
230    bool m_stopping;
231    bool m_autoStop;
232    size_t m_batchSize;
233};
234
235/// Automatic Scheduler switcher
236
237/// Automatically returns to Scheduler::getThis() when goes out of scope
238/// (by calling Scheduler::switchTo())
239struct SchedulerSwitcher : public boost::noncopyable
240{
241public:
242    /// Captures Scheduler::getThis(), and optionally calls target->switchTo()
243    /// if target != NULL
244    SchedulerSwitcher(Scheduler *target = NULL);
245    /// Calls switchTo() on the Scheduler captured in the constructor
246    /// @post Scheduler::getThis() == the Scheduler captured in the constructor
247    ~SchedulerSwitcher();
248
249private:
250    Scheduler *m_caller;
251};
252
253}
254
255#endif