IndexMap.h

// Copyright (C) 2015-2019 Chris Richardson, Garth N. Wells and Igor Baratta
//
// This file is part of DOLFINx (https://www.fenicsproject.org)
//
// SPDX-License-Identifier:    LGPL-3.0-or-later
 
#pragma once
 
#include <array>
#include <cstdint>
#include <dolfinx/common/MPI.h>
#include <dolfinx/graph/AdjacencyList.h>
#include <map>
#include <memory>
#include <tuple>
#include <utility>
#include <vector>
#include <xtl/xspan.hpp>
 
namespace dolfinx::common
{
// Forward declaration
class IndexMap;
 
std::tuple<std::int64_t, std::vector<std::int32_t>,
           std::vector<std::vector<std::int64_t>>,
           std::vector<std::vector<int>>>
stack_index_maps(
    const std::vector<
        std::pair<std::reference_wrapper<const common::IndexMap>, int>>& maps);
 
 
class IndexMap
{
public:
  enum class Mode
  {
    insert,
    add
  };
 
  enum class Direction
  {
    reverse,  // Ghost to owner
    forward,  // Owner to ghost
    symmetric // Symmetric. NOTE: To be removed
  };
 
  IndexMap(MPI_Comm comm, std::int32_t local_size);
 
  IndexMap(MPI_Comm mpi_comm, std::int32_t local_size,
           const std::vector<int>& dest_ranks,
           const std::vector<std::int64_t>& ghosts,
           const std::vector<int>& src_ranks);
 
  // Copy constructor
  IndexMap(const IndexMap& map) = delete;
 
  IndexMap(IndexMap&& map) = default;
 
  ~IndexMap() = default;
 
  IndexMap& operator=(IndexMap&& map) = default;
 
  // Copy assignment
  IndexMap& operator=(const IndexMap& map) = delete;
 
  std::array<std::int64_t, 2> local_range() const noexcept;
 
  std::int32_t num_ghosts() const noexcept;
 
  std::int32_t size_local() const noexcept;
 
  std::int64_t size_global() const noexcept;
 
  const std::vector<std::int64_t>& ghosts() const noexcept;
 
  MPI_Comm comm(Direction dir = Direction::symmetric) const;
 
  void local_to_global(const xtl::span<const std::int32_t>& local,
                       const xtl::span<std::int64_t>& global) const;
 
  void global_to_local(const xtl::span<const std::int64_t>& global,
                       const xtl::span<std::int32_t>& local) const;
 
  std::vector<std::int64_t> global_indices() const;
 
  const graph::AdjacencyList<std::int32_t>& shared_indices() const noexcept;
 
  std::vector<int> ghost_owner_rank() const;
 
  std::map<std::int32_t, std::set<int>> compute_shared_indices() const;
 
  template <typename T>
  void scatter_fwd(xtl::span<const T> local_data, xtl::span<T> remote_data,
                   int n) const;
 
  template <typename T>
  void scatter_rev(xtl::span<T> local_data, xtl::span<const T> remote_data,
                   int n, IndexMap::Mode op) const;
 
private:
  // Range of indices (global) owned by this process
  std::array<std::int64_t, 2> _local_range;
 
  // Number indices across communicator
  std::int64_t _size_global;
 
  // MPI neighborhood communicators
 
  // Communicator where the source ranks own the indices in the callers
  // halo, and the destination ranks 'ghost' indices owned by the
  // caller. I.e.,
  // - in-edges (src) are from ranks that own my ghosts
  // - out-edges (dest) go to ranks that 'ghost' my owned indices
  dolfinx::MPI::Comm _comm_owner_to_ghost;
 
  // Communicator where the source ranks have ghost indices that are
  // owned by the caller, and the destination ranks are the owners of
  // indices in the callers halo region. I.e.,
  // - in-edges (src) are from ranks that 'ghost' my owned indicies
  // - out-edges (dest) are to the owning ranks of my ghost indices
  dolfinx::MPI::Comm _comm_ghost_to_owner;
 
  // TODO: remove
  dolfinx::MPI::Comm _comm_symmetric;
 
  // Local-to-global map for ghost indices
  std::vector<std::int64_t> _ghosts;
 
  // Owning neighborhood rank (out edge) on '_comm_owner_to_ghost'
  // communicator for each ghost index
  std::vector<std::int32_t> _ghost_owners;
 
  // List of owned local indices that are in the halo (ghost) region on other
  // ranks, grouped by rank in the neighbor communicator (destination ranks in
  // forward communicator and source ranks in the reverse communicator).
  std::unique_ptr<graph::AdjacencyList<std::int32_t>> _shared_indices;
};
 
} // namespace dolfinx::common