/mordor/tests/fibersync.cpp

  1// Copyright (c) 2009 - Mozy, Inc.
  2
  3#include <boost/bind.hpp>
  4
  5#include "mordor/atomic.h"
  6#include "mordor/fiber.h"
  7#include "mordor/fibersynchronization.h"
  8#include "mordor/iomanager.h"
  9#include "mordor/sleep.h"
 10#include "mordor/test/test.h"
 11#include "mordor/workerpool.h"
 12
 13using namespace Mordor;
 14
 15template<typename M> void test_mutex_basic()
 16{
 17    WorkerPool pool;
 18    M mutex;
 19
 20    typename M::ScopedLock lock(mutex);
 21}
 22
 23MORDOR_UNITTEST(FiberMutex, basic)
 24{
 25    test_mutex_basic<FiberMutex>();
 26}
 27
 28template<typename M> inline void contentionFiber(int fiberNo, M &mutex, int &sequence)
 29{
 30    MORDOR_TEST_ASSERT_EQUAL(++sequence, fiberNo);
 31    typename M::ScopedLock lock(mutex);
 32    MORDOR_TEST_ASSERT_EQUAL(++sequence, fiberNo + 3 + 1);
 33}
 34
 35template<typename M> void test_mutex_contention()
 36{
 37    WorkerPool pool;
 38    M mutex;
 39    int sequence = 0;
 40    Fiber::ptr fiber1(new Fiber(NULL)), fiber2(new Fiber(NULL)),
 41        fiber3(new Fiber(NULL));
 42    fiber1->reset(boost::bind(&contentionFiber<M>, 1, boost::ref(mutex),
 43        boost::ref(sequence)));
 44    fiber2->reset(boost::bind(&contentionFiber<M>, 2, boost::ref(mutex),
 45        boost::ref(sequence)));
 46    fiber3->reset(boost::bind(&contentionFiber<M>, 3, boost::ref(mutex),
 47        boost::ref(sequence)));
 48
 49    {
 50        typename M::ScopedLock lock(mutex);
 51        pool.schedule(fiber1);
 52        pool.schedule(fiber2);
 53        pool.schedule(fiber3);
 54        pool.dispatch();
 55        MORDOR_TEST_ASSERT_EQUAL(++sequence, 4);
 56    }
 57    pool.dispatch();
 58    MORDOR_TEST_ASSERT_EQUAL(++sequence, 8);
 59}
 60
 61MORDOR_UNITTEST(FiberMutex, contention)
 62{
 63    test_mutex_contention<FiberMutex>();
 64}
 65
 66#ifndef NDEBUG
 67MORDOR_UNITTEST(FiberMutex, notRecursive)
 68{
 69    WorkerPool pool;
 70    FiberMutex mutex;
 71
 72    FiberMutex::ScopedLock lock(mutex);
 73    MORDOR_TEST_ASSERT_ASSERTED(mutex.lock());
 74}
 75#endif
 76
 77template<typename M> inline void lockIt(M &mutex)
 78{
 79    typename M::ScopedLock lock(mutex);
 80}
 81
 82template<typename M> void test_mutex_unlockUnique()
 83{
 84    WorkerPool pool;
 85    M mutex;
 86
 87    typename M::ScopedLock lock(mutex);
 88    MORDOR_TEST_ASSERT(!lock.unlockIfNotUnique());
 89    pool.schedule(boost::bind(&lockIt<M>, boost::ref(mutex)));
 90    Scheduler::yield();
 91    MORDOR_TEST_ASSERT(lock.unlockIfNotUnique());
 92    pool.dispatch();
 93}
 94
 95MORDOR_UNITTEST(FiberMutex, unlockUnique)
 96{
 97    test_mutex_unlockUnique<FiberMutex>();
 98}
 99
100template<typename M> inline void lockAndHold(IOManager &ioManager, M &mutex, Atomic<int> &counter)
101{
102    --counter;
103    typename M::ScopedLock lock(mutex);
104    while(counter > 0)
105        Mordor::sleep(ioManager, 50000); // sleep 50ms
106}
107
108template<typename M> void test_mutex_performance()
109{
110    IOManager ioManager(2, true);
111    M mutex;
112#ifdef X86_64
113#ifndef NDEBUG_PERF
114    int repeatness = 10000;
115#else
116    int repeatness = 50000;
117#endif
118#else
119    // on a 32bit system, a process can only have a 4GB virtual address
120    // each fiber wound take 1MB virtual address, this gives at most
121    // 4096 fibers can be alive simultaneously.
122    int repeatness = 1000;
123#endif
124    Atomic<int> counter = repeatness;
125    unsigned long long before = TimerManager::now();
126    for (int i=0; i<repeatness; ++i) {
127        ioManager.schedule(boost::bind(lockAndHold<M>,
128                                boost::ref(ioManager),
129                                boost::ref(mutex),
130                                boost::ref(counter)));
131    }
132    ioManager.stop();
133    unsigned long long elapse = TimerManager::now() - before;
134    MORDOR_LOG_INFO(Mordor::Log::root()) << "elapse: " << elapse;
135}
136
137MORDOR_UNITTEST(FiberMutex, mutexPerformance)
138{
139    test_mutex_performance<FiberMutex>();
140}
141
142MORDOR_UNITTEST(RecursiveFiberMutex, basic)
143{
144    test_mutex_basic<RecursiveFiberMutex>();
145}
146
147MORDOR_UNITTEST(RecursiveFiberMutex, recursive_basic)
148{
149    WorkerPool pool;
150    RecursiveFiberMutex mutex;
151
152    RecursiveFiberMutex::ScopedLock lock0(mutex);
153    {
154        RecursiveFiberMutex::ScopedLock lock1(mutex);
155        {
156            RecursiveFiberMutex::ScopedLock lock2(mutex);
157        }
158    }
159}
160
161MORDOR_UNITTEST(RecursiveFiberMutex, contention)
162{
163    test_mutex_contention<RecursiveFiberMutex>();
164}
165
166MORDOR_UNITTEST(RecursiveFiberMutex, mutexPerformance)
167{
168    test_mutex_performance<RecursiveFiberMutex>();
169}
170
171MORDOR_UNITTEST(RecursiveFiberMutex, unlockUnique)
172{
173    test_mutex_unlockUnique<RecursiveFiberMutex>();
174}
175
176static void signalMe(FiberCondition &condition, int &sequence)
177{
178    MORDOR_TEST_ASSERT_EQUAL(++sequence, 2);
179    condition.signal();
180}
181
182MORDOR_UNITTEST(FiberCondition, signal)
183{
184    int sequence = 0;
185    WorkerPool pool;
186    FiberMutex mutex;
187    FiberCondition condition(mutex);
188
189    FiberMutex::ScopedLock lock(mutex);
190    pool.schedule(boost::bind(&signalMe, boost::ref(condition),
191        boost::ref(sequence)));
192    MORDOR_TEST_ASSERT_EQUAL(++sequence, 1);
193    condition.wait();
194    MORDOR_TEST_ASSERT_EQUAL(++sequence, 3);
195}
196
197static void waitOnMe(FiberCondition &condition, FiberMutex &mutex,
198                     int &sequence, int expected)
199{
200    MORDOR_TEST_ASSERT_EQUAL(++sequence, expected * 2);
201    FiberMutex::ScopedLock lock(mutex);
202    MORDOR_TEST_ASSERT_EQUAL(++sequence, expected * 2 + 1);
203    condition.wait();
204    MORDOR_TEST_ASSERT_EQUAL(++sequence, expected + 8);
205}
206
207MORDOR_UNITTEST(FiberCondition, broadcast)
208{
209    int sequence = 0;
210    WorkerPool pool;
211    FiberMutex mutex;
212    FiberCondition condition(mutex);
213
214    pool.schedule(boost::bind(&waitOnMe, boost::ref(condition),
215        boost::ref(mutex), boost::ref(sequence), 1));
216    pool.schedule(boost::bind(&waitOnMe, boost::ref(condition),
217        boost::ref(mutex), boost::ref(sequence), 2));
218    pool.schedule(boost::bind(&waitOnMe, boost::ref(condition),
219        boost::ref(mutex), boost::ref(sequence), 3));
220    MORDOR_TEST_ASSERT_EQUAL(++sequence, 1);
221    pool.dispatch();
222    MORDOR_TEST_ASSERT_EQUAL(++sequence, 8);
223    condition.broadcast();
224    pool.dispatch();
225    MORDOR_TEST_ASSERT_EQUAL(++sequence, 12);
226}
227
228static void signalMe2(FiberEvent &event, int &sequence)
229{
230    MORDOR_TEST_ASSERT_EQUAL(++sequence, 2);
231    event.set();
232}
233
234MORDOR_UNITTEST(FiberEvent, autoResetSetWithoutExistingWaiters)
235{
236    int sequence = 0;
237    WorkerPool pool;
238    FiberEvent event;
239
240    pool.schedule(boost::bind(&signalMe2, boost::ref(event),
241        boost::ref(sequence)));
242    MORDOR_TEST_ASSERT_EQUAL(++sequence, 1);
243    event.wait();
244    MORDOR_TEST_ASSERT_EQUAL(++sequence, 3);
245    event.set();
246    // no fiber waiting at this moment, but the event is signaled
247    event.wait();
248    MORDOR_TEST_ASSERT_EQUAL(++sequence, 4);
249}
250
251static void signalMe3(TimerManager &manager, FiberEvent &event, FiberEvent &timer,
252                      int &sequence, unsigned long long awhile)
253{
254    MORDOR_TEST_ASSERT_EQUAL(++sequence, 2);
255    event.set();
256    timer.wait();
257    Mordor::sleep(manager, awhile);
258    MORDOR_TEST_ASSERT_EQUAL(++sequence, 4);
259    // a fiber is already waiting for a while
260    event.set();
261}
262
263MORDOR_UNITTEST(FiberEvent, autoResetSetWithExistingWaiters)
264{
265    int sequence = 0;
266    IOManager manager; // for a timer-enabled scheduler
267    FiberEvent event, timer;
268    static const unsigned long long awhile = 50000ULL; // sleep for 50ms
269
270    manager.schedule(boost::bind(&signalMe3, boost::ref(manager),
271        boost::ref(event), boost::ref(timer), boost::ref(sequence), awhile));
272    MORDOR_TEST_ASSERT_EQUAL(++sequence, 1);
273    event.wait();
274    MORDOR_TEST_ASSERT_EQUAL(++sequence, 3);
275    timer.set();
276    Test::TakesAtLeast _(awhile);
277    // the first set() call should not leave the event signaled, so this fiber
278    // should be blocked until signalMe3() allows it to move on.
279    event.wait();
280    MORDOR_TEST_ASSERT_EQUAL(++sequence, 5);
281}
282
283MORDOR_UNITTEST(FiberEvent, manualReset)
284{
285    int sequence = 0;
286    WorkerPool pool;
287    FiberEvent event(false);
288
289    pool.schedule(boost::bind(&signalMe2, boost::ref(event),
290        boost::ref(sequence)));
291    MORDOR_TEST_ASSERT_EQUAL(++sequence, 1);
292    event.wait();
293    MORDOR_TEST_ASSERT_EQUAL(++sequence, 3);
294    // It's manual reset; you can wait as many times as you want until it's
295    // reset
296    event.wait();
297    event.wait();
298}
299
300static void waitOnMe2(FiberEvent &event, int &sequence, int expected)
301{
302    MORDOR_TEST_ASSERT_EQUAL(++sequence, expected + 1);
303    event.wait();
304    MORDOR_TEST_ASSERT_EQUAL(++sequence, expected + 5);
305}
306
307MORDOR_UNITTEST(FiberEvent, manualResetMultiple)
308{
309    int sequence = 0;
310    WorkerPool pool;
311    FiberEvent event(false);
312
313    pool.schedule(boost::bind(&waitOnMe2, boost::ref(event),
314        boost::ref(sequence), 1));
315    pool.schedule(boost::bind(&waitOnMe2, boost::ref(event),
316        boost::ref(sequence), 2));
317    pool.schedule(boost::bind(&waitOnMe2, boost::ref(event),
318        boost::ref(sequence), 3));
319    MORDOR_TEST_ASSERT_EQUAL(++sequence, 1);
320    pool.dispatch();
321    MORDOR_TEST_ASSERT_EQUAL(++sequence, 5);
322    event.set();
323    pool.dispatch();
324    MORDOR_TEST_ASSERT_EQUAL(++sequence, 9);
325    // It's manual reset; you can wait as many times as you want until it's
326    // reset
327    event.wait();
328    event.wait();
329}
330
331class EventOwner
332{
333public:
334    EventOwner()
335        : m_event(false)
336        , m_destroying(false)
337    {}
338
339    ~EventOwner()
340    {
341
342        // 1 thread case - we can't awake from yielding in the wait() call
343        // until the fiber for the scheduled setEvent call is complete
344        //
345        // Multi-thread case - We can wake up because event is signalled, but
346        //other thread might still be inside m_event.set() call,
347        //with m_event mutex still locked.  Destroying that mutex while
348        //set() is being called can cause crash
349        m_event.wait();
350        m_destroying = true;
351
352        // Note: in debug mode the FiberEvent will get blocked waiting
353        // for the lock help in FiberEvent::set, but NDEBUG build will not
354    }
355
356    void setEvent()
357    {
358        m_event.set();
359        if (m_destroying) {
360            MORDOR_NOTREACHED();
361        }
362    }
363
364    FiberEvent m_event;
365    bool m_destroying;
366};
367
368
369MORDOR_UNITTEST(FiberEvent, destroyAfterSet)
370{
371    // Demo risk of using an FiberEvent in multi-threaded enviroment
372#if 0
373    {
374        //Not safe - owner destruction can start while pool2 is still
375        //executing setEvent().  Even though we wait on event the
376        //destructor is allowed to proceed before setEvent() has finished.
377        WorkerPool pool;
378        WorkerPool pool2(1,false);
379        EventOwner owner;
380        pool2.schedule(boost::bind(&EventOwner::setEvent, &owner));
381    }
382#endif
383
384    {
385        // Safe multi-threaded scenario - pool2 is stopped before event owner is destroyed
386        // which ensures that scheduled event is complete
387        WorkerPool pool;
388        WorkerPool pool2(1,false);
389        EventOwner owner;
390        pool2.schedule(boost::bind(&EventOwner::setEvent, &owner));
391        pool2.stop();
392    }
393
394    {
395        // Safe multi-threaded scenario - variables are declared in correct order so
396        // that pool2 is stopped before event owner is destroyed
397        WorkerPool pool;
398        EventOwner owner;
399        WorkerPool pool2(1,false);
400        pool2.schedule(boost::bind(&EventOwner::setEvent, &owner));
401    }
402
403    {
404        // Safe single threaded scenario - pool stops itself before
405        // owner is destroyed
406        WorkerPool pool;
407        EventOwner owner;
408        pool.schedule(boost::bind(&EventOwner::setEvent, &owner));
409        pool.stop();
410    }
411
412    {
413        // Safe single threaded scenario - pool destruction automatically
414        // blocks until setEvent is complete, then owner is destroyed
415        EventOwner owner;
416        WorkerPool pool;
417        pool.schedule(boost::bind(&EventOwner::setEvent, &owner));
418    }
419
420    {
421        // This is the only case that the event is actually needed and useful!
422        // Because only one fiber executes at a time on the single thread
423        WorkerPool pool;
424        EventOwner owner;
425        pool.schedule(boost::bind(&EventOwner::setEvent, &owner));
426    }
427
428
429}