/Src/Dependencies/Boost/boost/mpi/nonblocking.hpp

  1// Copyright (C) 2006 Douglas Gregor <doug.gregor -at- gmail.com>.
  2
  3// Use, modification and distribution is subject to the Boost Software
  4// License, Version 1.0. (See accompanying file LICENSE_1_0.txt or copy at
  5// http://www.boost.org/LICENSE_1_0.txt)
  6
  7/** @file nonblocking.hpp
  8 *
  9 *  This header defines operations for completing non-blocking
 10 *  communication requests.
 11 */
 12#ifndef BOOST_MPI_NONBLOCKING_HPP
 13#define BOOST_MPI_NONBLOCKING_HPP
 14
 15#include <boost/mpi/config.hpp>
 16#include <vector>
 17#include <iterator> // for std::iterator_traits
 18#include <boost/optional.hpp>
 19#include <utility> // for std::pair
 20#include <algorithm> // for iter_swap, reverse
 21#include <boost/static_assert.hpp>
 22#include <boost/mpi/request.hpp>
 23#include <boost/mpi/status.hpp>
 24#include <boost/mpi/exception.hpp>
 25
 26namespace boost { namespace mpi {
 27
 28/** 
 29 *  @brief Wait until any non-blocking request has completed.
 30 *
 31 *  This routine takes in a set of requests stored in the iterator
 32 *  range @c [first,last) and waits until any of these requests has
 33 *  been completed. It provides functionality equivalent to 
 34 *  @c MPI_Waitany.
 35 *
 36 *  @param first The iterator that denotes the beginning of the
 37 *  sequence of request objects.
 38 *
 39 *  @param last The iterator that denotes the end of the sequence of
 40 *  request objects. This may not be equal to @c first.
 41 *
 42 *  @returns A pair containing the status object that corresponds to
 43 *  the completed operation and the iterator referencing the completed
 44 *  request.
 45 */
 46template<typename ForwardIterator>
 47std::pair<status, ForwardIterator> 
 48wait_any(ForwardIterator first, ForwardIterator last)
 49{
 50  using std::advance;
 51
 52  BOOST_ASSERT(first != last);
 53  
 54  typedef typename std::iterator_traits<ForwardIterator>::difference_type
 55    difference_type;
 56
 57  bool all_trivial_requests = true;
 58  difference_type n = 0;
 59  ForwardIterator current = first;
 60  while (true) {
 61    // Check if we have found a completed request. If so, return it.
 62    if (optional<status> result = current->test())
 63      return std::make_pair(*result, current);
 64
 65    // Check if this request (and all others before it) are "trivial"
 66    // requests, e.g., they can be represented with a single
 67    // MPI_Request.
 68    all_trivial_requests = 
 69      all_trivial_requests
 70      && !current->m_handler 
 71      && current->m_requests[1] == MPI_REQUEST_NULL;
 72
 73    // Move to the next request.
 74    ++n;
 75    if (++current == last) {
 76      // We have reached the end of the list. If all requests thus far
 77      // have been trivial, we can call MPI_Waitany directly, because
 78      // it may be more efficient than our busy-wait semantics.
 79      if (all_trivial_requests) {
 80        std::vector<MPI_Request> requests;
 81        requests.reserve(n);
 82        for (current = first; current != last; ++current)
 83          requests.push_back(current->m_requests[0]);
 84
 85        // Let MPI wait until one of these operations completes.
 86        int index;
 87        status stat;
 88        BOOST_MPI_CHECK_RESULT(MPI_Waitany, 
 89                               (n, &requests[0], &index, &stat.m_status));
 90
 91        // We don't have a notion of empty requests or status objects,
 92        // so this is an error.
 93        if (index == MPI_UNDEFINED)
 94          boost::throw_exception(exception("MPI_Waitany", MPI_ERR_REQUEST));
 95
 96        // Find the iterator corresponding to the completed request.
 97        current = first;
 98        advance(current, index);
 99        current->m_requests[0] = requests[index];
100        return std::make_pair(stat, current);
101      }
102
103      // There are some nontrivial requests, so we must continue our
104      // busy waiting loop.
105      n = 0;
106      current = first;
107      all_trivial_requests = true;
108    }
109  }
110
111  // We cannot ever get here
112  BOOST_ASSERT(false);
113}
114
115/** 
116 *  @brief Test whether any non-blocking request has completed.
117 *
118 *  This routine takes in a set of requests stored in the iterator
119 *  range @c [first,last) and tests whether any of these requests has
120 *  been completed. This routine is similar to @c wait_any, but will
121 *  not block waiting for requests to completed. It provides
122 *  functionality equivalent to @c MPI_Testany.
123 *
124 *  @param first The iterator that denotes the beginning of the
125 *  sequence of request objects.
126 *
127 *  @param last The iterator that denotes the end of the sequence of
128 *  request objects. 
129 *
130 *  @returns If any outstanding requests have completed, a pair
131 *  containing the status object that corresponds to the completed
132 *  operation and the iterator referencing the completed
133 *  request. Otherwise, an empty @c optional<>.
134 */
135template<typename ForwardIterator>
136optional<std::pair<status, ForwardIterator> >
137test_any(ForwardIterator first, ForwardIterator last)
138{
139  for (ForwardIterator current = first; first != last; ++first) {
140    // Check if we have found a completed request. If so, return it.
141    if (optional<status> result = current->test())
142      return std::make_pair(*result, current);
143  }
144
145  // We found nothing
146  return optional<std::pair<status, ForwardIterator> >();
147}
148
149/** 
150 *  @brief Wait until all non-blocking requests have completed.
151 *
152 *  This routine takes in a set of requests stored in the iterator
153 *  range @c [first,last) and waits until all of these requests have
154 *  been completed. It provides functionality equivalent to 
155 *  @c MPI_Waitall.
156 *
157 *  @param first The iterator that denotes the beginning of the
158 *  sequence of request objects.
159 *
160 *  @param last The iterator that denotes the end of the sequence of
161 *  request objects. 
162 *
163 *  @param out If provided, an output iterator through which the
164 *  status of each request will be emitted. The @c status objects are
165 *  emitted in the same order as the requests are retrieved from 
166 *  @c [first,last).
167 *
168 *  @returns If an @p out parameter was provided, the value @c out
169 *  after all of the @c status objects have been emitted.
170 */
171template<typename ForwardIterator, typename OutputIterator>
172OutputIterator 
173wait_all(ForwardIterator first, ForwardIterator last, OutputIterator out)
174{
175  typedef typename std::iterator_traits<ForwardIterator>::difference_type
176    difference_type;
177
178  using std::distance;
179
180  difference_type num_outstanding_requests = distance(first, last);
181
182  std::vector<status> results(num_outstanding_requests);
183  std::vector<bool> completed(num_outstanding_requests);
184
185  while (num_outstanding_requests > 0) {
186    bool all_trivial_requests = true;
187    difference_type idx = 0;
188    for (ForwardIterator current = first; current != last; ++current, ++idx) {
189      if (!completed[idx]) {
190        if (optional<status> stat = current->test()) {
191          // This outstanding request has been completed. We're done.
192          results[idx] = *stat;
193          completed[idx] = true;
194          --num_outstanding_requests;
195          all_trivial_requests = false;
196        } else {
197          // Check if this request (and all others before it) are "trivial"
198          // requests, e.g., they can be represented with a single
199          // MPI_Request.
200          all_trivial_requests = 
201            all_trivial_requests
202            && !current->m_handler 
203            && current->m_requests[1] == MPI_REQUEST_NULL;          
204        }
205      }
206    }
207
208    // If we have yet to fulfill any requests and all of the requests
209    // are trivial (i.e., require only a single MPI_Request to be
210    // fulfilled), call MPI_Waitall directly.
211    if (all_trivial_requests 
212        && num_outstanding_requests == (difference_type)results.size()) {
213      std::vector<MPI_Request> requests;
214      requests.reserve(num_outstanding_requests);
215      for (ForwardIterator current = first; current != last; ++current)
216        requests.push_back(current->m_requests[0]);
217
218      // Let MPI wait until all of these operations completes.
219      std::vector<MPI_Status> stats(num_outstanding_requests);
220      BOOST_MPI_CHECK_RESULT(MPI_Waitall, 
221                             (num_outstanding_requests, &requests[0], 
222                              &stats[0]));
223
224      for (std::vector<MPI_Status>::iterator i = stats.begin(); 
225           i != stats.end(); ++i, ++out) {
226        status stat;
227        stat.m_status = *i;
228        *out = stat;
229      }
230
231      return out;
232    }
233
234    all_trivial_requests = false;
235  }
236
237  return std::copy(results.begin(), results.end(), out);
238}
239
240/**
241 * \overload
242 */
243template<typename ForwardIterator>
244void
245wait_all(ForwardIterator first, ForwardIterator last)
246{
247  typedef typename std::iterator_traits<ForwardIterator>::difference_type
248    difference_type;
249
250  using std::distance;
251
252  difference_type num_outstanding_requests = distance(first, last);
253
254  std::vector<bool> completed(num_outstanding_requests);
255
256  while (num_outstanding_requests > 0) {
257    bool all_trivial_requests = true;
258
259    difference_type idx = 0;
260    for (ForwardIterator current = first; current != last; ++current, ++idx) {
261      if (!completed[idx]) {
262        if (optional<status> stat = current->test()) {
263          // This outstanding request has been completed.
264          completed[idx] = true;
265          --num_outstanding_requests;
266          all_trivial_requests = false;
267        } else {
268          // Check if this request (and all others before it) are "trivial"
269          // requests, e.g., they can be represented with a single
270          // MPI_Request.
271          all_trivial_requests = 
272            all_trivial_requests
273            && !current->m_handler 
274            && current->m_requests[1] == MPI_REQUEST_NULL;          
275        }
276      }
277    }
278
279    // If we have yet to fulfill any requests and all of the requests
280    // are trivial (i.e., require only a single MPI_Request to be
281    // fulfilled), call MPI_Waitall directly.
282    if (all_trivial_requests 
283        && num_outstanding_requests == (difference_type)completed.size()) {
284      std::vector<MPI_Request> requests;
285      requests.reserve(num_outstanding_requests);
286      for (ForwardIterator current = first; current != last; ++current)
287        requests.push_back(current->m_requests[0]);
288
289      // Let MPI wait until all of these operations completes.
290      BOOST_MPI_CHECK_RESULT(MPI_Waitall, 
291                             (num_outstanding_requests, &requests[0], 
292                              MPI_STATUSES_IGNORE));
293
294      // Signal completion
295      num_outstanding_requests = 0;
296    }
297  }
298}
299
300/** 
301 *  @brief Tests whether all non-blocking requests have completed.
302 *
303 *  This routine takes in a set of requests stored in the iterator
304 *  range @c [first,last) and determines whether all of these requests
305 *  have been completed. However, due to limitations of the underlying
306 *  MPI implementation, if any of the requests refers to a
307 *  non-blocking send or receive of a serialized data type, @c
308 *  test_all will always return the equivalent of @c false (i.e., the
309 *  requests cannot all be finished at this time). This routine
310 *  performs the same functionality as @c wait_all, except that this
311 *  routine will not block. This routine provides functionality
312 *  equivalent to @c MPI_Testall.
313 *
314 *  @param first The iterator that denotes the beginning of the
315 *  sequence of request objects.
316 *
317 *  @param last The iterator that denotes the end of the sequence of
318 *  request objects. 
319 *
320 *  @param out If provided and all requests hav been completed, an
321 *  output iterator through which the status of each request will be
322 *  emitted. The @c status objects are emitted in the same order as
323 *  the requests are retrieved from @c [first,last).
324 *
325 *  @returns If an @p out parameter was provided, the value @c out
326 *  after all of the @c status objects have been emitted (if all
327 *  requests were completed) or an empty @c optional<>. If no @p out
328 *  parameter was provided, returns @c true if all requests have
329 *  completed or @c false otherwise.
330 */
331template<typename ForwardIterator, typename OutputIterator>
332optional<OutputIterator>
333test_all(ForwardIterator first, ForwardIterator last, OutputIterator out)
334{
335  std::vector<MPI_Request> requests;
336  for (; first != last; ++first) {
337    // If we have a non-trivial request, then no requests can be
338    // completed.
339    if (first->m_handler || first->m_requests[1] != MPI_REQUEST_NULL)
340      return optional<OutputIterator>();
341
342    requests.push_back(first->m_requests[0]);
343  }
344
345  int flag = 0;
346  int n = requests.size();
347  std::vector<MPI_Status> stats(n);
348  BOOST_MPI_CHECK_RESULT(MPI_Testall, (n, &requests[0], &flag, &stats[0]));
349  if (flag) {
350    for (int i = 0; i < n; ++i, ++out) {
351      status stat;
352      stat.m_status = stats[i];
353      *out = stat;
354    }
355    return out;
356  } else {
357    return optional<OutputIterator>();
358  }
359}
360
361/**
362 *  \overload
363 */
364template<typename ForwardIterator>
365bool
366test_all(ForwardIterator first, ForwardIterator last)
367{
368  std::vector<MPI_Request> requests;
369  for (; first != last; ++first) {
370    // If we have a non-trivial request, then no requests can be
371    // completed.
372    if (first->m_handler || first->m_requests[1] != MPI_REQUEST_NULL)
373      return false;
374
375    requests.push_back(first->m_requests[0]);
376  }
377
378  int flag = 0;
379  int n = requests.size();
380  BOOST_MPI_CHECK_RESULT(MPI_Testall, 
381                         (n, &requests[0], &flag, MPI_STATUSES_IGNORE));
382  return flag != 0;
383}
384
385/** 
386 *  @brief Wait until some non-blocking requests have completed.
387 *
388 *  This routine takes in a set of requests stored in the iterator
389 *  range @c [first,last) and waits until at least one of the requests
390 *  has completed. It then completes all of the requests it can,
391 *  partitioning the input sequence into pending requests followed by
392 *  completed requests. If an output iterator is provided, @c status
393 *  objects will be emitted for each of the completed requests. This
394 *  routine provides functionality equivalent to @c MPI_Waitsome.
395 *
396 *  @param first The iterator that denotes the beginning of the
397 *  sequence of request objects.
398 *
399 *  @param last The iterator that denotes the end of the sequence of
400 *  request objects. This may not be equal to @c first.
401 *
402 *  @param out If provided, the @c status objects corresponding to
403 *  completed requests will be emitted through this output iterator.
404
405 *  @returns If the @p out parameter was provided, a pair containing
406 *  the output iterator @p out after all of the @c status objects have
407 *  been written through it and an iterator referencing the first
408 *  completed request. If no @p out parameter was provided, only the
409 *  iterator referencing the first completed request will be emitted.
410 */
411template<typename BidirectionalIterator, typename OutputIterator>
412std::pair<OutputIterator, BidirectionalIterator> 
413wait_some(BidirectionalIterator first, BidirectionalIterator last,
414          OutputIterator out)
415{
416  using std::advance;
417
418  if (first == last)
419    return std::make_pair(out, first);
420  
421  typedef typename std::iterator_traits<BidirectionalIterator>::difference_type
422    difference_type;
423
424  bool all_trivial_requests = true;
425  difference_type n = 0;
426  BidirectionalIterator current = first;
427  BidirectionalIterator start_of_completed = last;
428  while (true) {
429    // Check if we have found a completed request. 
430    if (optional<status> result = current->test()) {
431      using std::iter_swap;
432
433      // Emit the resulting status object
434      *out++ = *result;
435
436      // We're expanding the set of completed requests
437      --start_of_completed;
438
439      if (current == start_of_completed) {
440        // If we have hit the end of the list of pending
441        // requests. Finish up by fixing the order of the completed
442        // set to match the order in which we emitted status objects,
443        // then return.
444        std::reverse(start_of_completed, last);
445        return std::make_pair(out, start_of_completed);
446      }
447
448      // Swap the request we just completed with the last request that
449      // has not yet been tested.
450      iter_swap(current, start_of_completed);
451
452      continue;
453    }
454
455    // Check if this request (and all others before it) are "trivial"
456    // requests, e.g., they can be represented with a single
457    // MPI_Request.
458    all_trivial_requests = 
459      all_trivial_requests
460      && !current->m_handler 
461      && current->m_requests[1] == MPI_REQUEST_NULL;
462
463    // Move to the next request.
464    ++n;
465    if (++current == start_of_completed) {
466      if (start_of_completed != last) {
467        // We have satisfied some requests. Make the order of the
468        // completed requests match that of the status objects we've
469        // already emitted and we're done.
470        std::reverse(start_of_completed, last);
471        return std::make_pair(out, start_of_completed);
472      }
473
474      // We have reached the end of the list. If all requests thus far
475      // have been trivial, we can call MPI_Waitsome directly, because
476      // it may be more efficient than our busy-wait semantics.
477      if (all_trivial_requests) {
478        std::vector<MPI_Request> requests;
479        std::vector<int> indices(n);
480        std::vector<MPI_Status> stats(n);
481        requests.reserve(n);
482        for (current = first; current != last; ++current)
483          requests.push_back(current->m_requests[0]);
484
485        // Let MPI wait until some of these operations complete.
486        int num_completed;
487        BOOST_MPI_CHECK_RESULT(MPI_Waitsome, 
488                               (n, &requests[0], &num_completed, &indices[0],
489                                &stats[0]));
490
491        // Translate the index-based result of MPI_Waitsome into a
492        // partitioning on the requests.
493        int current_offset = 0;
494        current = first;
495        for (int index = 0; index < num_completed; ++index, ++out) {
496          using std::iter_swap;
497
498          // Move "current" to the request object at this index
499          advance(current, indices[index] - current_offset);
500          current_offset = indices[index];
501
502          // Emit the status object
503          status stat;
504          stat.m_status = stats[index];
505          *out = stat;
506
507          // Finish up the request and swap it into the "completed
508          // requests" partition.
509          current->m_requests[0] = requests[indices[index]];
510          --start_of_completed;
511          iter_swap(current, start_of_completed);
512        }
513
514        // We have satisfied some requests. Make the order of the
515        // completed requests match that of the status objects we've
516        // already emitted and we're done.
517        std::reverse(start_of_completed, last);
518        return std::make_pair(out, start_of_completed);
519      }
520
521      // There are some nontrivial requests, so we must continue our
522      // busy waiting loop.
523      n = 0;
524      current = first;
525    }
526  }
527
528  // We cannot ever get here
529  BOOST_ASSERT(false);
530}
531
532/**
533 *  \overload
534 */
535template<typename BidirectionalIterator>
536BidirectionalIterator
537wait_some(BidirectionalIterator first, BidirectionalIterator last)
538{
539  using std::advance;
540
541  if (first == last)
542    return first;
543  
544  typedef typename std::iterator_traits<BidirectionalIterator>::difference_type
545    difference_type;
546
547  bool all_trivial_requests = true;
548  difference_type n = 0;
549  BidirectionalIterator current = first;
550  BidirectionalIterator start_of_completed = last;
551  while (true) {
552    // Check if we have found a completed request. 
553    if (optional<status> result = current->test()) {
554      using std::iter_swap;
555
556      // We're expanding the set of completed requests
557      --start_of_completed;
558
559      // If we have hit the end of the list of pending requests, we're
560      // done.
561      if (current == start_of_completed)
562        return start_of_completed;
563
564      // Swap the request we just completed with the last request that
565      // has not yet been tested.
566      iter_swap(current, start_of_completed);
567
568      continue;
569    }
570
571    // Check if this request (and all others before it) are "trivial"
572    // requests, e.g., they can be represented with a single
573    // MPI_Request.
574    all_trivial_requests = 
575      all_trivial_requests
576      && !current->m_handler 
577      && current->m_requests[1] == MPI_REQUEST_NULL;
578
579    // Move to the next request.
580    ++n;
581    if (++current == start_of_completed) {
582        // If we have satisfied some requests, we're done.
583      if (start_of_completed != last)
584        return start_of_completed;
585
586      // We have reached the end of the list. If all requests thus far
587      // have been trivial, we can call MPI_Waitsome directly, because
588      // it may be more efficient than our busy-wait semantics.
589      if (all_trivial_requests) {
590        std::vector<MPI_Request> requests;
591        std::vector<int> indices(n);
592        requests.reserve(n);
593        for (current = first; current != last; ++current)
594          requests.push_back(current->m_requests[0]);
595
596        // Let MPI wait until some of these operations complete.
597        int num_completed;
598        BOOST_MPI_CHECK_RESULT(MPI_Waitsome, 
599                               (n, &requests[0], &num_completed, &indices[0],
600                                MPI_STATUSES_IGNORE));
601
602        // Translate the index-based result of MPI_Waitsome into a
603        // partitioning on the requests.
604        int current_offset = 0;
605        current = first;
606        for (int index = 0; index < num_completed; ++index) {
607          using std::iter_swap;
608
609          // Move "current" to the request object at this index
610          advance(current, indices[index] - current_offset);
611          current_offset = indices[index];
612
613          // Finish up the request and swap it into the "completed
614          // requests" partition.
615          current->m_requests[0] = requests[indices[index]];
616          --start_of_completed;
617          iter_swap(current, start_of_completed);
618        }
619
620        // We have satisfied some requests, so we are done.
621        return start_of_completed;
622      }
623
624      // There are some nontrivial requests, so we must continue our
625      // busy waiting loop.
626      n = 0;
627      current = first;
628    }
629  }
630
631  // We cannot ever get here
632  BOOST_ASSERT(false);
633}
634
635/** 
636 *  @brief Test whether some non-blocking requests have completed.
637 *
638 *  This routine takes in a set of requests stored in the iterator
639 *  range @c [first,last) and tests to see if any of the requests has
640 *  completed. It completes all of the requests it can, partitioning
641 *  the input sequence into pending requests followed by completed
642 *  requests. If an output iterator is provided, @c status objects
643 *  will be emitted for each of the completed requests. This routine
644 *  is similar to @c wait_some, but does not wait until any requests
645 *  have completed. This routine provides functionality equivalent to
646 *  @c MPI_Testsome.
647 *
648 *  @param first The iterator that denotes the beginning of the
649 *  sequence of request objects.
650 *
651 *  @param last The iterator that denotes the end of the sequence of
652 *  request objects. This may not be equal to @c first.
653 *
654 *  @param out If provided, the @c status objects corresponding to
655 *  completed requests will be emitted through this output iterator.
656
657 *  @returns If the @p out parameter was provided, a pair containing
658 *  the output iterator @p out after all of the @c status objects have
659 *  been written through it and an iterator referencing the first
660 *  completed request. If no @p out parameter was provided, only the
661 *  iterator referencing the first completed request will be emitted.
662 */
663template<typename BidirectionalIterator, typename OutputIterator>
664std::pair<OutputIterator, BidirectionalIterator> 
665test_some(BidirectionalIterator first, BidirectionalIterator last,
666          OutputIterator out)
667{
668  BidirectionalIterator current = first;
669  BidirectionalIterator start_of_completed = last;
670  while (current != start_of_completed) {
671    // Check if we have found a completed request. 
672    if (optional<status> result = current->test()) {
673      using std::iter_swap;
674
675      // Emit the resulting status object
676      *out++ = *result;
677
678      // We're expanding the set of completed requests
679      --start_of_completed;
680
681      // Swap the request we just completed with the last request that
682      // has not yet been tested.
683      iter_swap(current, start_of_completed);
684
685      continue;
686    }
687
688    // Move to the next request.
689    ++current;
690  }
691
692  // Finish up by fixing the order of the completed set to match the
693  // order in which we emitted status objects, then return.
694  std::reverse(start_of_completed, last);
695  return std::make_pair(out, start_of_completed);
696}
697
698/**
699 *  \overload
700 */
701template<typename BidirectionalIterator>
702BidirectionalIterator
703test_some(BidirectionalIterator first, BidirectionalIterator last)
704{
705  BidirectionalIterator current = first;
706  BidirectionalIterator start_of_completed = last;
707  while (current != start_of_completed) {
708    // Check if we have found a completed request. 
709    if (optional<status> result = current->test()) {
710      using std::iter_swap;
711
712      // We're expanding the set of completed requests
713      --start_of_completed;
714
715      // Swap the request we just completed with the last request that
716      // has not yet been tested.
717      iter_swap(current, start_of_completed);
718
719      continue;
720    }
721
722    // Move to the next request.
723    ++current;
724  }
725
726  return start_of_completed;
727}
728
729} } // end namespace boost::mpi
730
731
732#endif // BOOST_MPI_NONBLOCKING_HPP