// Copyright (C) 2004-2006 The Trustees of Indiana University.
// Copyright (C) 2007 Douglas Gregor 

// Use, modification and distribution is subject to the Boost Software
// License, Version 1.0. (See accompanying file LICENSE_1_0.txt or copy at
// http://www.boost.org/LICENSE_1_0.txt)

//  Authors: Douglas Gregor
//           Andrew Lumsdaine

#ifndef BOOST_GRAPH_DISTRIBUTED_ADJACENCY_LIST_HPP
#define BOOST_GRAPH_DISTRIBUTED_ADJACENCY_LIST_HPP

#ifndef BOOST_GRAPH_USE_MPI
#error "Parallel BGL files should not be included unless <boost/graph/use_mpi.hpp> has been included"
#endif

#include <boost/graph/adjacency_list.hpp>
#include <boost/graph/properties.hpp>
#include <boost/graph/graph_traits.hpp>
#include <boost/graph/iteration_macros.hpp>
#include <boost/graph/distributed/concepts.hpp>
#include <boost/iterator/transform_iterator.hpp>
#include <boost/property_map/property_map.hpp>
#include <boost/graph/adjacency_iterator.hpp>
#include <boost/property_map/parallel/distributed_property_map.hpp>
#include <boost/property_map/parallel/local_property_map.hpp>
#include <boost/graph/parallel/detail/property_holders.hpp>
#include <boost/mpl/if.hpp>
#include <boost/type_traits/is_same.hpp>
#include <boost/assert.hpp>
#include <list>
#include <algorithm>
#include <boost/limits.hpp>
#include <boost/graph/parallel/properties.hpp>
#include <boost/graph/parallel/distribution.hpp>
#include <boost/graph/parallel/algorithm.hpp>
#include <boost/graph/distributed/selector.hpp>
#include <boost/graph/parallel/process_group.hpp>

// Callbacks
#include <boost/function/function2.hpp>

// Serialization
#include <boost/serialization/base_object.hpp>
#include <boost/mpi/datatype.hpp>
#include <boost/pending/property_serialize.hpp>
#include <boost/graph/distributed/unsafe_serialize.hpp>

// Named vertices
#include <boost/graph/distributed/named_graph.hpp>

#include <boost/graph/distributed/shuffled_distribution.hpp>

namespace boost {

  /// The type used to store an identifier that uniquely names a processor.
  // NGE: I doubt we'll be running on more than 32768 procs for the time being
  typedef /*int*/ short processor_id_type;

  // Tell which processor the target of an edge resides on (for
  // directed graphs) or which processor the other end point of the
  // edge resides on (for undirected graphs).
  enum edge_target_processor_id_t { edge_target_processor_id };
  BOOST_INSTALL_PROPERTY(edge, target_processor_id);

  // For undirected graphs, tells whether the edge is locally owned.
  enum edge_locally_owned_t { edge_locally_owned };
  BOOST_INSTALL_PROPERTY(edge, locally_owned);

  // For bidirectional graphs, stores the incoming edges.
  enum vertex_in_edges_t { vertex_in_edges };
  BOOST_INSTALL_PROPERTY(vertex, in_edges);

  /// Tag class for directed, distributed adjacency list
  struct directed_distributed_adj_list_tag
    : public virtual distributed_graph_tag,
      public virtual distributed_vertex_list_graph_tag,
      public virtual distributed_edge_list_graph_tag,
      public virtual incidence_graph_tag,
      public virtual adjacency_graph_tag {};

  /// Tag class for bidirectional, distributed adjacency list
  struct bidirectional_distributed_adj_list_tag
    : public virtual distributed_graph_tag,
      public virtual distributed_vertex_list_graph_tag,
      public virtual distributed_edge_list_graph_tag,
      public virtual incidence_graph_tag,
      public virtual adjacency_graph_tag,
      public virtual bidirectional_graph_tag {};

  /// Tag class for undirected, distributed adjacency list
  struct undirected_distributed_adj_list_tag
    : public virtual distributed_graph_tag,
      public virtual distributed_vertex_list_graph_tag,
      public virtual distributed_edge_list_graph_tag,
      public virtual incidence_graph_tag,
      public virtual adjacency_graph_tag,
      public virtual bidirectional_graph_tag {};

  namespace detail {
    template<typename Archiver, typename Directed, typename Vertex>
    void
    serialize(Archiver& ar, edge_base<Directed, Vertex>& e,
              const unsigned int /*version*/)
    {
      ar & unsafe_serialize(e.m_source)
         & unsafe_serialize(e.m_target);
    }

    template<typename Archiver, typename Directed, typename Vertex>
    void
    serialize(Archiver& ar, edge_desc_impl<Directed, Vertex>& e,
              const unsigned int /*version*/)
    {
      ar & boost::serialization::base_object<edge_base<Directed, Vertex> >(e)
         & unsafe_serialize(e.m_eproperty);
    }
  }

  namespace detail { namespace parallel {
  
    /**
     * A distributed vertex descriptor. These descriptors contain both
     * the ID of the processor that owns the vertex and a local vertex
     * descriptor that identifies the particular vertex for that
     * processor.
     */
    template<typename LocalDescriptor>
    struct global_descriptor
    {
      typedef LocalDescriptor local_descriptor_type;

      global_descriptor() : owner(), local() { }

      global_descriptor(processor_id_type owner, LocalDescriptor local)
        : owner(owner), local(local) { }

      processor_id_type owner;
      LocalDescriptor local;

      /**
       * A function object that, given a processor ID, generates
       * distributed vertex descriptors from local vertex
       * descriptors. This function object is used by the
       * vertex_iterator of the distributed adjacency list.
       */
      struct generator
      {
        typedef global_descriptor<LocalDescriptor> result_type;
        typedef LocalDescriptor argument_type;

        generator() {}
        generator(processor_id_type owner) : owner(owner) {}

        result_type operator()(argument_type v) const
        { return result_type(owner, v); }

      private:
        processor_id_type owner;
      };

      template<typename Archiver>
      void serialize(Archiver& ar, const unsigned int /*version*/)
      {
        ar & owner & unsafe_serialize(local);
      }
    };

    /// Determine the process that owns the given descriptor
    template<typename LocalDescriptor>
    inline processor_id_type owner(const global_descriptor<LocalDescriptor>& v)
    { return v.owner; }

    /// Determine the local portion of the given descriptor
    template<typename LocalDescriptor>
    inline LocalDescriptor local(const global_descriptor<LocalDescriptor>& v)
    { return v.local; }

    /// Compare distributed vertex descriptors for equality
    template<typename LocalDescriptor>
    inline bool
    operator==(const global_descriptor<LocalDescriptor>& u,
               const global_descriptor<LocalDescriptor>& v)
    {
      return u.owner == v.owner && u.local == v.local;
    }

    /// Compare distributed vertex descriptors for inequality
    template<typename LocalDescriptor>
    inline bool
    operator!=(const global_descriptor<LocalDescriptor>& u,
               const global_descriptor<LocalDescriptor>& v)
    { return !(u == v); }

    template<typename LocalDescriptor>
    inline bool
    operator<(const global_descriptor<LocalDescriptor>& u,
              const global_descriptor<LocalDescriptor>& v)
    {
      return (u.owner) < v.owner || (u.owner == v.owner && (u.local) < v.local);
    }

    template<typename LocalDescriptor>
    inline bool
    operator<=(const global_descriptor<LocalDescriptor>& u,
               const global_descriptor<LocalDescriptor>& v)
    {
      return u.owner <= v.owner || (u.owner == v.owner && u.local <= v.local);
    }

    template<typename LocalDescriptor>
    inline bool
    operator>(const global_descriptor<LocalDescriptor>& u,
              const global_descriptor<LocalDescriptor>& v)
    {
      return v < u;
    }

    template<typename LocalDescriptor>
    inline bool
    operator>=(const global_descriptor<LocalDescriptor>& u,
               const global_descriptor<LocalDescriptor>& v)
    {
      return v <= u;
    }

    // DPG TBD: Add <, <=, >=, > for global descriptors

    /**
     * A Readable Property Map that extracts a global descriptor pair
     * from a global_descriptor.
     */
    template<typename LocalDescriptor>
    struct global_descriptor_property_map
    {
      typedef std::pair<processor_id_type, LocalDescriptor> value_type;
      typedef value_type reference;
      typedef global_descriptor<LocalDescriptor> key_type;
      typedef readable_property_map_tag category;
    };

    template<typename LocalDescriptor>
    inline std::pair<processor_id_type, LocalDescriptor>
    get(global_descriptor_property_map<LocalDescriptor>,
        global_descriptor<LocalDescriptor> x)
    {
      return std::pair<processor_id_type, LocalDescriptor>(x.owner, x.local);
    }

    /**
     * A Readable Property Map that extracts the owner of a global
     * descriptor.
     */
    template<typename LocalDescriptor>
    struct owner_property_map
    {
      typedef processor_id_type value_type;
      typedef value_type reference;
      typedef global_descriptor<LocalDescriptor> key_type;
      typedef readable_property_map_tag category;
    };

    template<typename LocalDescriptor>
    inline processor_id_type
    get(owner_property_map<LocalDescriptor>,
        global_descriptor<LocalDescriptor> x)
    {
      return x.owner;
    }

    /**
     * A Readable Property Map that extracts the local descriptor from
     * a global descriptor.
     */
    template<typename LocalDescriptor>
    struct local_descriptor_property_map
    {
      typedef LocalDescriptor value_type;
      typedef value_type reference;
      typedef global_descriptor<LocalDescriptor> key_type;
      typedef readable_property_map_tag category;
    };

    template<typename LocalDescriptor>
    inline LocalDescriptor
    get(local_descriptor_property_map<LocalDescriptor>,
        global_descriptor<LocalDescriptor> x)
    {
      return x.local;
    }

    /**
     * Stores an incoming edge for a bidirectional distributed
     * adjacency list. The user does not see this type directly,
     * because it is just an implementation detail.
     */
    template<typename Edge>
    struct stored_in_edge
    {
      stored_in_edge(processor_id_type sp, Edge e)
        : source_processor(sp), e(e) {}

      processor_id_type source_processor;
      Edge e;
    };

    /**
     * A distributed edge descriptor. These descriptors contain the
     * underlying edge descriptor, the processor IDs for both the
     * source and the target of the edge, and a boolean flag that
     * indicates which of the processors actually owns the edge.
     */
    template<typename Edge>
    struct edge_descriptor
    {
      edge_descriptor(processor_id_type sp = processor_id_type(),
                      processor_id_type tp = processor_id_type(),
                      bool owns = false, Edge ld = Edge())
        : source_processor(sp), target_processor(tp),
          source_owns_edge(owns), local(ld) {}

      processor_id_type owner() const
      {
        return source_owns_edge? source_processor : target_processor;
      }

      /// The processor that the source vertex resides on
      processor_id_type source_processor;

      /// The processor that the target vertex resides on
      processor_id_type target_processor;

      /// True when the source processor owns the edge, false when the
      /// target processor owns the edge.
      bool source_owns_edge;

      /// The local edge descriptor.
      Edge local;

      /**
       * Function object that generates edge descriptors for the
       * out_edge_iterator of the given distributed adjacency list
       * from the edge descriptors of the underlying adjacency list.
       */
      template<typename Graph>
      class out_generator
      {
        typedef typename Graph::directed_selector directed_selector;

      public:
        typedef edge_descriptor<Edge> result_type;
        typedef Edge argument_type;

        out_generator() : g(0) {}
        explicit out_generator(const Graph& g) : g(&g) {}

        result_type operator()(argument_type e) const
        { return map(e, directed_selector()); }

      private:
        result_type map(argument_type e, directedS) const
        {
          return result_type(g->processor(),
                             get(edge_target_processor_id, g->base(), e),
                             true, e);
        }

        result_type map(argument_type e, bidirectionalS) const
        {
          return result_type(g->processor(),
                             get(edge_target_processor_id, g->base(), e),
                             true, e);
        }

        result_type map(argument_type e, undirectedS) const
        {
          return result_type(g->processor(),
                             get(edge_target_processor_id, g->base(), e),
                             get(edge_locally_owned, g->base(), e),
                             e);
        }

        const Graph* g;
      };

      /**
       * Function object that generates edge descriptors for the
       * in_edge_iterator of the given distributed adjacency list
       * from the edge descriptors of the underlying adjacency list.
       */
      template<typename Graph>
      class in_generator
      {
        typedef typename Graph::directed_selector DirectedS;

      public:
        typedef typename boost::mpl::if_<is_same<DirectedS, bidirectionalS>,
                                         stored_in_edge<Edge>,
                                         Edge>::type argument_type;
        typedef edge_descriptor<Edge> result_type;

        in_generator() : g(0) {}
        explicit in_generator(const Graph& g) : g(&g) {}

        result_type operator()(argument_type e) const
        { return map(e, DirectedS()); }

      private:
        /**
         * For a bidirectional graph, we just generate the appropriate
         * edge. No tricks.
         */
        result_type map(argument_type e, bidirectionalS) const
        {
          return result_type(e.source_processor,
                             g->processor(),
                             true,
                             e.e);
        }

        /**
         * For an undirected graph, we generate descriptors for the
         * incoming edges by swapping the source/target of the
         * underlying edge descriptor (a hack). The target processor
         * ID on the edge is actually the source processor for this
         * edge, and our processor is the target processor. If the
         * edge is locally owned, then it is owned by the target (us);
         * otherwise it is owned by the source.
         */
        result_type map(argument_type e, undirectedS) const
        {
          typename Graph::local_edge_descriptor local_edge(e);
          // TBD: This is a very, VERY lame hack that takes advantage
          // of our knowledge of the internals of the BGL
          // adjacency_list. There should be a cleaner way to handle
          // this...
          using std::swap;
          swap(local_edge.m_source, local_edge.m_target);
          return result_type(get(edge_target_processor_id, g->base(), e),
                             g->processor(),
                             !get(edge_locally_owned, g->base(), e),
                             local_edge);
        }

        const Graph* g;
      };

    private:
      friend class boost::serialization::access;

      template<typename Archiver>
      void serialize(Archiver& ar, const unsigned int /*version*/)
      {
        ar
          & source_processor
          & target_processor
          & source_owns_edge
          & local;
      }
    };

    /// Determine the process that owns this edge
    template<typename Edge>
    inline processor_id_type
    owner(const edge_descriptor<Edge>& e)
    { return e.source_owns_edge? e.source_processor : e.target_processor; }

    /// Determine the local descriptor for this edge.
    template<typename Edge>
    inline Edge
    local(const edge_descriptor<Edge>& e)
    { return e.local; }

    /**
     * A Readable Property Map that extracts the owner and local
     * descriptor of an edge descriptor.
     */
    template<typename Edge>
    struct edge_global_property_map
    {
      typedef std::pair<processor_id_type, Edge> value_type;
      typedef value_type reference;
      typedef edge_descriptor<Edge> key_type;
      typedef readable_property_map_tag category;
    };

    template<typename Edge>
    inline std::pair<processor_id_type, Edge>
    get(edge_global_property_map<Edge>, const edge_descriptor<Edge>& e)
    {
      typedef std::pair<processor_id_type, Edge> result_type;
      return result_type(e.source_owns_edge? e.source_processor
                         /* target owns edge*/: e.target_processor,
                         e.local);
    }

    /**
     * A Readable Property Map that extracts the owner of an edge
     * descriptor.
     */
    template<typename Edge>
    struct edge_owner_property_map
    {
      typedef processor_id_type value_type;
      typedef value_type reference;
      typedef edge_descriptor<Edge> key_type;
      typedef readable_property_map_tag category;
    };

    template<typename Edge>
    inline processor_id_type
    get(edge_owner_property_map<Edge>, const edge_descriptor<Edge>& e)
    {
      return e.source_owns_edge? e.source_processor : e.target_processor;
    }

    /**
     * A Readable Property Map that extracts the local descriptor from
     * an edge descriptor.
     */
    template<typename Edge>
    struct edge_local_property_map
    {
      typedef Edge value_type;
      typedef value_type reference;
      typedef edge_descriptor<Edge> key_type;
      typedef readable_property_map_tag category;
    };

    template<typename Edge>
    inline Edge
    get(edge_local_property_map<Edge>,
        const edge_descriptor<Edge>& e)
    {
      return e.local;
    }

    /** Compare distributed edge descriptors for equality.
     *
     * \todo need edge_descriptor to know if it is undirected so we
     * can compare both ways.
     */
    template<typename Edge>
    inline bool
    operator==(const edge_descriptor<Edge>& e1,
               const edge_descriptor<Edge>& e2)
    {
      return (e1.source_processor == e2.source_processor
              && e1.target_processor == e2.target_processor
              && e1.local == e2.local);
    }

    /// Compare distributed edge descriptors for inequality.
    template<typename Edge>
    inline bool
    operator!=(const edge_descriptor<Edge>& e1,
               const edge_descriptor<Edge>& e2)
    { return !(e1 == e2); }

    /**
     * Configuration for the distributed adjacency list. We use this
     * parameter to store all of the configuration details for the
     * implementation of the distributed adjacency list, which allows us to
     * get at the distribution type in the maybe_named_graph.
     */
    template<typename OutEdgeListS, typename ProcessGroup,
             typename InVertexListS, typename InDistribution,
             typename DirectedS, typename VertexProperty, 
             typename EdgeProperty, typename GraphProperty, 
             typename EdgeListS>
    struct adjacency_list_config
    {
      typedef typename mpl::if_<is_same<InVertexListS, defaultS>, 
                                vecS, InVertexListS>::type 
        VertexListS;

      /// Introduce the target processor ID property for each edge
      typedef property<edge_target_processor_id_t, processor_id_type,
                       EdgeProperty> edge_property_with_id;

      /// For undirected graphs, introduce the locally-owned property for edges
      typedef typename boost::mpl::if_<is_same<DirectedS, undirectedS>,
                                       property<edge_locally_owned_t, bool,
                                                edge_property_with_id>,
                                       edge_property_with_id>::type
        base_edge_property_type;

      /// The edge descriptor type for the local subgraph
      typedef typename adjacency_list_traits<OutEdgeListS,
                                             VertexListS,
                                             directedS>::edge_descriptor
        local_edge_descriptor;

      /// For bidirectional graphs, the type of an incoming stored edge
      typedef stored_in_edge<local_edge_descriptor> bidir_stored_edge;

      /// The container type that will store incoming edges for a
      /// bidirectional graph.
      typedef typename container_gen<EdgeListS, bidir_stored_edge>::type
        in_edge_list_type;

      // Bidirectional graphs have an extra vertex property to store
      // the incoming edges.
      typedef typename boost::mpl::if_<is_same<DirectedS, bidirectionalS>,
                                       property<vertex_in_edges_t, in_edge_list_type,
                                                VertexProperty>,
                                       VertexProperty>::type 
        base_vertex_property_type;

      // The type of the distributed adjacency list
      typedef adjacency_list<OutEdgeListS,
                             distributedS<ProcessGroup, 
                                          VertexListS, 
                                          InDistribution>,
                             DirectedS, VertexProperty, EdgeProperty,
                             GraphProperty, EdgeListS> 
        graph_type;

      // The type of the underlying adjacency list implementation
      typedef adjacency_list<OutEdgeListS, VertexListS, directedS,
                             base_vertex_property_type,
                             base_edge_property_type,
                             GraphProperty,
                             EdgeListS> 
        inherited;
      
      typedef InDistribution in_distribution_type;
      typedef typename inherited::vertices_size_type vertices_size_type;

          typedef typename ::boost::graph::distributed::select_distribution<
              in_distribution_type, VertexProperty, vertices_size_type, 
              ProcessGroup>::type 
        base_distribution_type;

          typedef ::boost::graph::distributed::shuffled_distribution<
          base_distribution_type> distribution_type;

      typedef VertexProperty vertex_property_type;
      typedef EdgeProperty edge_property_type;
      typedef ProcessGroup process_group_type;

      typedef VertexListS vertex_list_selector;
      typedef OutEdgeListS out_edge_list_selector;
      typedef DirectedS directed_selector;
      typedef GraphProperty graph_property_type;
      typedef EdgeListS edge_list_selector;
    };

    // Maybe initialize the indices of each vertex
    template<typename IteratorPair, typename VertexIndexMap>
    void
    maybe_initialize_vertex_indices(IteratorPair p, VertexIndexMap to_index,
                                    read_write_property_map_tag)
    {
      typedef typename property_traits<VertexIndexMap>::value_type index_t;
      index_t next_index = 0;
      while (p.first != p.second)
        put(to_index, *p.first++, next_index++);
    }

    template<typename IteratorPair, typename VertexIndexMap>
    inline void
    maybe_initialize_vertex_indices(IteratorPair p, VertexIndexMap to_index,
                                    readable_property_map_tag)
    {
      // Do nothing
    }

    template<typename IteratorPair, typename VertexIndexMap>
    inline void
    maybe_initialize_vertex_indices(IteratorPair p, VertexIndexMap to_index)
    {
      typedef typename property_traits<VertexIndexMap>::category category;
      maybe_initialize_vertex_indices(p, to_index, category());
    }

    template<typename IteratorPair>
    inline void
    maybe_initialize_vertex_indices(IteratorPair p,
                                    ::boost::detail::error_property_not_found)
    { }

    /***********************************************************************
     * Message Payloads                                                    *
     ***********************************************************************/

    /**
     * Data stored with a msg_add_edge message, which requests the
     * remote addition of an edge.
     */
    template<typename Vertex, typename LocalVertex>
    struct msg_add_edge_data
    {
      msg_add_edge_data() { }

      msg_add_edge_data(Vertex source, Vertex target)
        : source(source.local), target(target) { }

      /// The source of the edge; the processor will be the
      /// receiving processor.
      LocalVertex source;
        
      /// The target of the edge.
      Vertex target;
        
      template<typename Archiver>
      void serialize(Archiver& ar, const unsigned int /*version*/)
      {
        ar & unsafe_serialize(source) & target;
      }
    };

    /**
     * Like @c msg_add_edge_data, but also includes a user-specified
     * property value to be attached to the edge.
     */
    template<typename Vertex, typename LocalVertex, typename EdgeProperty>
    struct msg_add_edge_with_property_data
      : msg_add_edge_data<Vertex, LocalVertex>, 
        maybe_store_property<EdgeProperty>
    {
    private:
      typedef msg_add_edge_data<Vertex, LocalVertex> inherited_data;
      typedef maybe_store_property<EdgeProperty> inherited_property;

    public:
      msg_add_edge_with_property_data() { }

      msg_add_edge_with_property_data(Vertex source, 
                                      Vertex target,
                                      const EdgeProperty& property)
        : inherited_data(source, target),
          inherited_property(property) { }
      
      template<typename Archiver>
      void serialize(Archiver& ar, const unsigned int /*version*/)
      {
        ar & boost::serialization::base_object<inherited_data>(*this) 
           & boost::serialization::base_object<inherited_property>(*this);
      }
    };

    //------------------------------------------------------------------------
    // Distributed adjacency list property map details
    /**
     * Metafunction that extracts the given property from the given
     * distributed adjacency list type. This could be implemented much
     * more cleanly, but even newer versions of GCC (e.g., 3.2.3)
     * cannot properly handle partial specializations involving
     * enumerator types.
     */
    template<typename Property>
    struct get_adj_list_pmap
    {
      template<typename Graph>
      struct apply
      {
        typedef Graph graph_type;
        typedef typename graph_type::process_group_type process_group_type;
        typedef typename graph_type::inherited base_graph_type;
        typedef typename property_map<base_graph_type, Property>::type
          local_pmap;
        typedef typename property_map<base_graph_type, Property>::const_type
          local_const_pmap;

        typedef graph_traits<graph_type> traits;
        typedef typename graph_type::local_vertex_descriptor local_vertex;
        typedef typename property_traits<local_pmap>::key_type local_key_type;

        typedef typename property_traits<local_pmap>::value_type value_type;

        typedef typename property_map<Graph, vertex_global_t>::const_type
          vertex_global_map;
        typedef typename property_map<Graph, edge_global_t>::const_type
          edge_global_map;

        typedef typename mpl::if_c<(is_same<local_key_type,
                                            local_vertex>::value),
                                   vertex_global_map, edge_global_map>::type
          global_map;

      public:
        typedef ::boost::parallel::distributed_property_map<
                  process_group_type, global_map, local_pmap> type;

        typedef ::boost::parallel::distributed_property_map<
                  process_group_type, global_map, local_const_pmap> const_type;
      };
    };

    /**
     * The local vertex index property map is actually a mapping from
     * the local vertex descriptors to vertex indices.
     */
    template<>
    struct get_adj_list_pmap<vertex_local_index_t>
    {
      template<typename Graph>
      struct apply
        : ::boost::property_map<typename Graph::inherited, vertex_index_t>
      { };
    };

    /**
     * The vertex index property map maps from global descriptors
     * (e.g., the vertex descriptor of a distributed adjacency list)
     * to the underlying local index. It is not valid to use this
     * property map with nonlocal descriptors.
     */
    template<>
    struct get_adj_list_pmap<vertex_index_t>
    {
      template<typename Graph>
      struct apply
      {
      private:
        typedef typename property_map<Graph, vertex_global_t>::const_type
          global_map;

        typedef property_map<typename Graph::inherited, vertex_index_t> local;

      public:
        typedef local_property_map<typename Graph::process_group_type,
                                   global_map,
                                   typename local::type> type;
        typedef local_property_map<typename Graph::process_group_type,
                                   global_map,
                                   typename local::const_type> const_type;
      };
    };

    /**
     * The vertex owner property map maps from vertex descriptors to
     * the processor that owns the vertex.
     */
    template<>
    struct get_adj_list_pmap<vertex_global_t>
    {
      template<typename Graph>
      struct apply
      {
      private:
        typedef typename Graph::local_vertex_descriptor
          local_vertex_descriptor;
      public:
        typedef global_descriptor_property_map<local_vertex_descriptor> type;
        typedef type const_type;
      };
    };

    /**
     * The vertex owner property map maps from vertex descriptors to
     * the processor that owns the vertex.
     */
    template<>
    struct get_adj_list_pmap<vertex_owner_t>
    {
      template<typename Graph>
      struct apply
      {
      private:
        typedef typename Graph::local_vertex_descriptor
          local_vertex_descriptor;
      public:
        typedef owner_property_map<local_vertex_descriptor> type;
        typedef type const_type;
      };
    };

    /**
     * The vertex local property map maps from vertex descriptors to
     * the local descriptor for that vertex.
     */
    template<>
    struct get_adj_list_pmap<vertex_local_t>
    {
      template<typename Graph>
      struct apply
      {
      private:
        typedef typename Graph::local_vertex_descriptor
          local_vertex_descriptor;
      public:
        typedef local_descriptor_property_map<local_vertex_descriptor> type;
        typedef type const_type;
      };
    };

    /**
     * The edge global property map maps from edge descriptors to
     * a pair of the owning processor and local descriptor.
     */
    template<>
    struct get_adj_list_pmap<edge_global_t>
    {
      template<typename Graph>
      struct apply
      {
      private:
        typedef typename Graph::local_edge_descriptor
          local_edge_descriptor;
      public:
        typedef edge_global_property_map<local_edge_descriptor> type;
        typedef type const_type;
      };
    };

    /**
     * The edge owner property map maps from edge descriptors to
     * the processor that owns the edge.
     */
    template<>
    struct get_adj_list_pmap<edge_owner_t>
    {
      template<typename Graph>
      struct apply
      {
      private:
        typedef typename Graph::local_edge_descriptor
          local_edge_descriptor;
      public:
        typedef edge_owner_property_map<local_edge_descriptor> type;
        typedef type const_type;
      };
    };

    /**
     * The edge local property map maps from edge descriptors to
     * the local descriptor for that edge.
     */
    template<>
    struct get_adj_list_pmap<edge_local_t>
    {
      template<typename Graph>
      struct apply
      {
      private:
        typedef typename Graph::local_edge_descriptor
          local_edge_descriptor;
      public:
        typedef edge_local_property_map<local_edge_descriptor> type;
        typedef type const_type;
      };
    };
    //------------------------------------------------------------------------

    // Directed graphs do not have in edges, so this is a no-op
    template<typename Graph>
    inline void
    remove_in_edge(typename Graph::edge_descriptor, Graph&, directedS)
    { }

    // Bidirectional graphs have in edges stored in the
    // vertex_in_edges property.
    template<typename Graph>
    inline void
    remove_in_edge(typename Graph::edge_descriptor e, Graph& g, bidirectionalS)
    {
      typedef typename Graph::in_edge_list_type in_edge_list_type;
      in_edge_list_type& in_edges =
        get(vertex_in_edges, g.base())[target(e, g).local];
      typename in_edge_list_type::iterator i = in_edges.begin();
      while (i != in_edges.end()
             && !(i->source_processor == source(e, g).owner)
             && i->e == e.local)
        ++i;

      BOOST_ASSERT(i != in_edges.end());
      in_edges.erase(i);
    }

    // Undirected graphs have in edges stored as normal edges.
    template<typename Graph>
    inline void
    remove_in_edge(typename Graph::edge_descriptor e, Graph& g, undirectedS)
    {
      typedef typename Graph::inherited base_type;
      typedef typename graph_traits<Graph>::vertex_descriptor vertex_descriptor;

      // TBD: can we make this more efficient?
      // Removing edge (v, u). v is local
      base_type& bg = g.base();
      vertex_descriptor u = source(e, g);
      vertex_descriptor v = target(e, g);
      if (v.owner != process_id(g.process_group())) {
        using std::swap;
        swap(u, v);
      }

      typename graph_traits<base_type>::out_edge_iterator ei, ei_end;
      for (boost::tie(ei, ei_end) = out_edges(v.local, bg); ei != ei_end; ++ei)
      {
        if (target(*ei, g.base()) == u.local
            // TBD: deal with parallel edges properly && *ei == e
            && get(edge_target_processor_id, bg, *ei) == u.owner) {
          remove_edge(ei, bg);
          return;
        }
      }

      if (v.owner == process_id(g.process_group())) {

      }
    }

    //------------------------------------------------------------------------
    // Lazy addition of edges

    // Work around the fact that an adjacency_list with vecS vertex
    // storage automatically adds edges when the descriptor is
    // out-of-range.
    template <class Graph, class Config, class Base>
    inline std::pair<typename Config::edge_descriptor, bool>
    add_local_edge(typename Config::vertex_descriptor u,
                   typename Config::vertex_descriptor v,
                   const typename Config::edge_property_type& p,
                   vec_adj_list_impl<Graph, Config, Base>& g_)
    {
      adj_list_helper<Config, Base>& g = g_;
      return add_edge(u, v, p, g);
    }

    template <class Graph, class Config, class Base>
    inline std::pair<typename Config::edge_descriptor, bool>
    add_local_edge(typename Config::vertex_descriptor u,
                   typename Config::vertex_descriptor v,
                   const typename Config::edge_property_type& p,
                   boost::adj_list_impl<Graph, Config, Base>& g)
    {
      return add_edge(u, v, p, g);
    }

    template <class EdgeProperty,class EdgeDescriptor>
    struct msg_nonlocal_edge_data
      : public detail::parallel::maybe_store_property<EdgeProperty>
    {
      typedef EdgeProperty edge_property_type;
      typedef EdgeDescriptor local_edge_descriptor;
      typedef detail::parallel::maybe_store_property<edge_property_type> 
        inherited;

      msg_nonlocal_edge_data() {}
      msg_nonlocal_edge_data(local_edge_descriptor e,
                             const edge_property_type& p)
        : inherited(p), e(e) { }

      local_edge_descriptor e;

      template<typename Archiver>
      void serialize(Archiver& ar, const unsigned int /*version*/)
      {
        ar & boost::serialization::base_object<inherited>(*this) & e;
      }
    };

    template <class EdgeDescriptor>
    struct msg_remove_edge_data
    {
      typedef EdgeDescriptor edge_descriptor;
      msg_remove_edge_data() {}
      explicit msg_remove_edge_data(edge_descriptor e) : e(e) {}

      edge_descriptor e;

      template<typename Archiver>
      void serialize(Archiver& ar, const unsigned int /*version*/)
      {
        ar & e;
      }
    };

  } } // end namespace detail::parallel

  /**
   * Adjacency list traits for a distributed adjacency list. Contains
   * the vertex and edge descriptors, the directed-ness, and the
   * parallel edges typedefs.
   */
  template<typename OutEdgeListS, typename ProcessGroup,
           typename InVertexListS, typename InDistribution, typename DirectedS>
  struct adjacency_list_traits<OutEdgeListS,
                               distributedS<ProcessGroup, 
                                            InVertexListS,
                                            InDistribution>,
                               DirectedS>
  {
  private:
    typedef typename mpl::if_<is_same<InVertexListS, defaultS>,
                              vecS,
                              InVertexListS>::type VertexListS;

    typedef adjacency_list_traits<OutEdgeListS, VertexListS, directedS>
      base_type;

  public:
    typedef typename base_type::vertex_descriptor local_vertex_descriptor;
    typedef typename base_type::edge_descriptor   local_edge_descriptor;

    typedef typename boost::mpl::if_<typename DirectedS::is_bidir_t,
      bidirectional_tag,
      typename boost::mpl::if_<typename DirectedS::is_directed_t,
        directed_tag, undirected_tag
      >::type
    >::type directed_category;

    typedef typename parallel_edge_traits<OutEdgeListS>::type
      edge_parallel_category;

    typedef detail::parallel::global_descriptor<local_vertex_descriptor>
      vertex_descriptor;

    typedef detail::parallel::edge_descriptor<local_edge_descriptor>
      edge_descriptor;
  };

#define PBGL_DISTRIB_ADJLIST_TEMPLATE_PARMS                                    \
  typename OutEdgeListS, typename ProcessGroup, typename InVertexListS,        \
  typename InDistribution, typename DirectedS, typename VertexProperty,        \
  typename EdgeProperty,  typename GraphProperty, typename EdgeListS

#define PBGL_DISTRIB_ADJLIST_TYPE                                              \
  adjacency_list<OutEdgeListS,                                                 \
                 distributedS<ProcessGroup, InVertexListS, InDistribution>,    \
                 DirectedS, VertexProperty, EdgeProperty, GraphProperty,       \
                 EdgeListS>

#define PBGL_DISTRIB_ADJLIST_TEMPLATE_PARMS_CONFIG                             \
  typename OutEdgeListS, typename ProcessGroup, typename InVertexListS,        \
  typename InDistribution, typename VertexProperty,                            \
  typename EdgeProperty,  typename GraphProperty, typename EdgeListS
  
#define PBGL_DISTRIB_ADJLIST_TYPE_CONFIG(directed)                             \
  adjacency_list<OutEdgeListS,                                                 \
                 distributedS<ProcessGroup, InVertexListS, InDistribution>,    \
                 directed, VertexProperty, EdgeProperty, GraphProperty,        \
                 EdgeListS>
                 
  /** A distributed adjacency list.
   *
   * This class template partial specialization defines a distributed
   * (or "partitioned") adjacency list graph. The distributed
   * adjacency list is similar to the standard Boost Graph Library
   * adjacency list, which stores a list of vertices and for each
   * verted the list of edges outgoing from the vertex (and, in some
   * cases, also the edges incoming to the vertex). The distributed
   * adjacency list differs in that it partitions the graph into
   * several subgraphs that are then divided among different
   * processors (or nodes within a cluster). The distributed adjacency
   * list attempts to maintain a high degree of compatibility with the
   * standard, non-distributed adjacency list.
   *
   * The graph is partitioned by vertex, with each processor storing
   * all of the required information for a particular subset of the
   * vertices, including vertex properties, outgoing edges, and (for
   * bidirectional graphs) incoming edges. This information is
   * accessible only on the processor that owns the vertex: for
   * instance, if processor 0 owns vertex @c v, no other processor can
   * directly access the properties of @c v or enumerate its outgoing
   * edges.
   *
   * Edges in a graph may be entirely local (connecting two local
   * vertices), but more often it is the case that edges are
   * non-local, meaning that the two vertices they connect reside in
   * different processes. Edge properties are stored with the
   * originating vertex for directed and bidirectional graphs, and are
   * therefore only accessible from the processor that owns the
   * originating vertex. Other processors may query the source and
   * target of the edge, but cannot access its properties. This is
   * particularly interesting when accessing the incoming edges of a
   * bidirectional graph, which are not guaranteed to be stored on the
   * processor that is able to perform the iteration. For undirected
   * graphs the situation is more complicated, since no vertex clearly
   * owns the edges: the list of edges incident to a vertex may
   * contain a mix of local and non-local edges.
   *
   * The distributed adjacency list is able to model several of the
   * existing Graph concepts. It models the Graph concept because it
   * exposes vertex and edge descriptors in the normal way; these
   * descriptors model the GlobalDescriptor concept (because they have
   * an owner and a local descriptor), and as such the distributed
   * adjacency list models the DistributedGraph concept. The adjacency
   * list also models the IncidenceGraph and AdjacencyGraph concepts,
   * although this is only true so long as the domain of the valid
   * expression arguments are restricted to vertices and edges stored
   * locally. Likewise, bidirectional and undirected distributed
   * adjacency lists model the BidirectionalGraph concept (vertex and
   * edge domains must be respectived) and the distributed adjacency
   * list models the MutableGraph concept (vertices and edges can only
   * be added or removed locally). T he distributed adjacency list
   * does not, however, model the VertexListGraph or EdgeListGraph
   * concepts, because we can not efficiently enumerate all vertices
   * or edges in the graph. Instead, the local subsets of vertices and
   * edges can be enumerated (with the same syntax): the distributed
   * adjacency list therefore models the DistributedVertexListGraph
   * and DistributedEdgeListGraph concepts, because concurrent
   * iteration over all of the vertices or edges stored on each
   * processor will visit each vertex or edge.
   *
   * The distributed adjacency list is distinguished from the
   * non-distributed version by the vertex list descriptor, which will
   * be @c distributedS<ProcessGroup,VertexListS>. Here,
   * the VertexListS type plays the same role as the VertexListS type
   * in the non-distributed adjacency list: it allows one to select
   * the data structure that will be used to store the local
   * vertices. The ProcessGroup type, on the other hand, is unique to
   * distributed data structures: it is the type that abstracts a
   * group of cooperating processes, and it used for process
   * identification, communication, and synchronization, among other
   * things. Different process group types represent different
   * communication mediums (e.g., MPI, PVM, TCP) or different models
   * of communication (LogP, CGM, BSP, synchronous, etc.). This
   * distributed adjacency list assumes a model based on non-blocking
   * sends.
   *
   * Distribution of vertices across different processors is
   * accomplished in two different ways. When initially constructing
   * the graph, the user may provide a distribution object (that
   * models the Distribution concept), which will determine the
   * distribution of vertices to each process. Additionally, the @c
   * add_vertex and @c add_edge operations add vertices or edges
   * stored on the local processor. For @c add_edge, this is
   * accomplished by requiring that the source vertex of the new edge
   * be local to the process executing @c add_edge.
   *
   * Internal properties of a distributed adjacency list are
   * accessible in the same manner as internal properties for a
   * non-distributed adjacency list for local vertices or
   * edges. Access to properties for remote vertices or edges occurs
   * with the same syntax, but involve communication with the owner of
   * the information: for more information, refer to class template
   * @ref distributed_property_map, which manages distributed
   * property maps. Note that the distributed property maps created
   * for internal properties determine their reduction operation via
   * the metafunction @ref property_reduce, which for the vast
   * majority of uses is correct behavior.
   *
   * Communication among the processes coordinating on a particular
   * distributed graph relies on non-blocking message passing along
   * with synchronization. Local portions of the distributed graph may
   * be modified concurrently, including the introduction of non-local
   * edges, but prior to accessing the graph it is recommended that
   * the @c synchronize free function be invoked on the graph to clear
   * up any pending interprocess communication and modifications. All
   * processes will then be released from the synchronization barrier
   * concurrently.
   *
   * \todo Determine precisely what we should do with nonlocal edges
   * in undirected graphs. Our parallelization of certain algorithms
   * relies on the ability to access edge property maps immediately
   * (e.g., edge_weight_t), so it may be necessary to duplicate the
   * edge properties in both processes (but then we need some form of
   * coherence protocol).
   *
   * \todo What does the user do if @c property_reduce doesn't do the
   * right thing?
   */
  template<typename OutEdgeListS, typename ProcessGroup,
           typename InVertexListS, typename InDistribution, typename DirectedS,
           typename VertexProperty, typename EdgeProperty, 
           typename GraphProperty, typename EdgeListS>
  class adjacency_list<OutEdgeListS,
                       distributedS<ProcessGroup, 
                                    InVertexListS, 
                                    InDistribution>,
                       DirectedS, VertexProperty,
                       EdgeProperty, GraphProperty, EdgeListS>
    : // Support for named vertices
      public graph::distributed::maybe_named_graph<   
        adjacency_list<OutEdgeListS,
                       distributedS<ProcessGroup,
                                    InVertexListS,
                                    InDistribution>,
                       DirectedS, VertexProperty,
                       EdgeProperty, GraphProperty, EdgeListS>,
        typename adjacency_list_traits<OutEdgeListS, 
                                       distributedS<ProcessGroup,
                                                    InVertexListS,
                                                    InDistribution>,
                                       DirectedS>::vertex_descriptor,
        typename adjacency_list_traits<OutEdgeListS, 
                                       distributedS<ProcessGroup,
                                                    InVertexListS,
                                                    InDistribution>,
                                       DirectedS>::edge_descriptor,
        detail::parallel::adjacency_list_config<OutEdgeListS, ProcessGroup, 
                                                InVertexListS, InDistribution,
                                                DirectedS, VertexProperty, 
                                                EdgeProperty, GraphProperty, 
                                                EdgeListS> >
  {
    typedef detail::parallel::adjacency_list_config<OutEdgeListS, ProcessGroup, 
                                                InVertexListS, InDistribution,
                                                DirectedS, VertexProperty, 
                                                EdgeProperty, GraphProperty, 
                                                EdgeListS>
      config_type;
      
    typedef adjacency_list_traits<OutEdgeListS,
                                  distributedS<ProcessGroup, 
                                               InVertexListS, 
                                               InDistribution>,
                                  DirectedS> 
      traits_type;

    typedef typename DirectedS::is_directed_t is_directed;

    typedef EdgeListS edge_list_selector;

  public:
    /// The container type that will store incoming edges for a
    /// bidirectional graph.
    typedef typename config_type::in_edge_list_type in_edge_list_type;
//    typedef typename inherited::edge_descriptor   edge_descriptor;

    /// The type of the underlying adjacency list implementation
    typedef typename config_type::inherited inherited;

    /// The type of properties stored in the local subgraph
    /// Bidirectional graphs have an extra vertex property to store
    /// the incoming edges.
    typedef typename inherited::vertex_property_type
      base_vertex_property_type;

    /// The type of the distributed adjacency list (this type)
    typedef typename config_type::graph_type graph_type;

    /// Expose graph components and graph category
    typedef typename traits_type::local_vertex_descriptor
      local_vertex_descriptor;
    typedef typename traits_type::local_edge_descriptor
      local_edge_descriptor;
    typedef typename traits…