40 #ifndef TPETRA_DISTRIBUTOR_HPP
41 #define TPETRA_DISTRIBUTOR_HPP
43 #include "Tpetra_Details_DistributorActor.hpp"
47 #include "Teuchos_as.hpp"
48 #include "Teuchos_Describable.hpp"
49 #include "Teuchos_ParameterListAcceptorDefaultBase.hpp"
50 #include "Teuchos_VerboseObject.hpp"
53 #include "KokkosCompat_View.hpp"
54 #include "Kokkos_Core.hpp"
55 #include "Kokkos_TeuchosCommAdapters.hpp"
58 #include <type_traits>
133 public Teuchos::Describable,
134 public Teuchos::ParameterListAcceptorDefaultBase {
147 explicit Distributor (
const Teuchos::RCP<
const Teuchos::Comm<int> >& comm);
160 Distributor (
const Teuchos::RCP<
const Teuchos::Comm<int> >& comm,
161 const Teuchos::RCP<Teuchos::FancyOStream>& out);
176 Distributor (
const Teuchos::RCP<
const Teuchos::Comm<int> >& comm,
177 const Teuchos::RCP<Teuchos::ParameterList>& plist);
195 Distributor (
const Teuchos::RCP<
const Teuchos::Comm<int> >& comm,
196 const Teuchos::RCP<Teuchos::FancyOStream>& out,
197 const Teuchos::RCP<Teuchos::ParameterList>& plist);
254 size_t createFromSends (
const Teuchos::ArrayView<const int>& exportProcIDs);
289 template <
class Ordinal>
292 const Teuchos::ArrayView<const int>& remoteProcIDs,
293 Teuchos::Array<Ordinal>& exportIDs,
294 Teuchos::Array<int>& exportProcIDs);
305 const Teuchos::ArrayView<const int>& remoteProcIDs);
340 Teuchos::ArrayView<const int>
getProcsTo()
const;
365 return plan_.howInitialized();
382 Teuchos::RCP<Distributor>
getReverse(
bool create=
true)
const;
424 template <
class ExpView,
class ImpView>
425 typename std::enable_if<(Kokkos::is_view<ExpView>::value && Kokkos::is_view<ImpView>::value)>::type
427 const ExpView &exports,
429 const ImpView &imports);
452 template <
class ExpView,
class ImpView>
453 typename std::enable_if<(Kokkos::is_view<ExpView>::value && Kokkos::is_view<ImpView>::value)>::type
455 const Teuchos::ArrayView<const size_t>& numExportPacketsPerLID,
456 const ImpView &imports,
457 const Teuchos::ArrayView<const size_t>& numImportPacketsPerLID);
483 template <
class ExpView,
class ImpView>
484 typename std::enable_if<(Kokkos::is_view<ExpView>::value && Kokkos::is_view<ImpView>::value)>::type
485 doPosts (
const ExpView &exports,
487 const ImpView &imports);
507 template <
class ExpView,
class ImpView>
508 typename std::enable_if<(Kokkos::is_view<ExpView>::value && Kokkos::is_view<ImpView>::value)>::type
509 doPosts (
const ExpView &exports,
510 const Teuchos::ArrayView<const size_t>& numExportPacketsPerLID,
511 const ImpView &imports,
512 const Teuchos::ArrayView<const size_t>& numImportPacketsPerLID);
518 template <
class ExpView,
class ImpView>
519 typename std::enable_if<(Kokkos::is_view<ExpView>::value && Kokkos::is_view<ImpView>::value)>::type
522 const ImpView &imports);
528 template <
class ExpView,
class ImpView>
529 typename std::enable_if<(Kokkos::is_view<ExpView>::value && Kokkos::is_view<ImpView>::value)>::type
531 const Teuchos::ArrayView<const size_t>& numExportPacketsPerLID,
532 const ImpView &imports,
533 const Teuchos::ArrayView<const size_t>& numImportPacketsPerLID);
539 template <
class ExpView,
class ImpView>
540 typename std::enable_if<(Kokkos::is_view<ExpView>::value && Kokkos::is_view<ImpView>::value)>::type
543 const ImpView &imports);
549 template <
class ExpView,
class ImpView>
550 typename std::enable_if<(Kokkos::is_view<ExpView>::value && Kokkos::is_view<ImpView>::value)>::type
552 const Teuchos::ArrayView<const size_t>& numExportPacketsPerLID,
553 const ImpView &imports,
554 const Teuchos::ArrayView<const size_t>& numImportPacketsPerLID);
585 describe (Teuchos::FancyOStream& out,
586 const Teuchos::EVerbosityLevel verbLevel =
587 Teuchos::Describable::verbLevel_default)
const;
597 Details::DistributorActor actor_;
603 static bool getVerbose();
609 std::unique_ptr<std::string>
610 createPrefix(
const char methodName[])
const;
613 bool verbose_ = getVerbose();
620 mutable Teuchos::RCP<Distributor> reverseDistributor_;
634 template <
class Ordinal>
635 void computeSends (
const Teuchos::ArrayView<const Ordinal> &remoteGIDs,
636 const Teuchos::ArrayView<const int> &remoteProcIDs,
637 Teuchos::Array<Ordinal> &exportGIDs,
638 Teuchos::Array<int> &exportProcIDs);
641 void createReverseDistributor()
const;
649 localDescribeToString (
const Teuchos::EVerbosityLevel vl)
const;
652 template <
class ExpView,
class ImpView>
653 typename std::enable_if<(Kokkos::is_view<ExpView>::value && Kokkos::is_view<ImpView>::value)>::type
655 doPostsAndWaits (
const ExpView& exports,
657 const ImpView& imports)
659 actor_.doPostsAndWaits(plan_, exports, numPackets, imports);
662 template <
class ExpView,
class ImpView>
663 typename std::enable_if<(Kokkos::is_view<ExpView>::value && Kokkos::is_view<ImpView>::value)>::type
665 doPostsAndWaits(
const ExpView& exports,
666 const Teuchos::ArrayView<const size_t>& numExportPacketsPerLID,
667 const ImpView& imports,
668 const Teuchos::ArrayView<const size_t>& numImportPacketsPerLID)
670 actor_.
doPostsAndWaits(plan_, exports, numExportPacketsPerLID, imports, numImportPacketsPerLID);
674 template <
class ExpView,
class ImpView>
675 typename std::enable_if<(Kokkos::is_view<ExpView>::value && Kokkos::is_view<ImpView>::value)>::type
677 doPosts (
const ExpView &exports,
679 const ImpView &imports)
681 actor_.
doPosts(plan_, exports, numPackets, imports);
684 template <
class ExpView,
class ImpView>
685 typename std::enable_if<(Kokkos::is_view<ExpView>::value && Kokkos::is_view<ImpView>::value)>::type
687 doPosts (
const ExpView &exports,
688 const Teuchos::ArrayView<const size_t>& numExportPacketsPerLID,
689 const ImpView &imports,
690 const Teuchos::ArrayView<const size_t>& numImportPacketsPerLID)
692 actor_.
doPosts(plan_, exports, numExportPacketsPerLID, imports, numImportPacketsPerLID);
695 template <
class ExpView,
class ImpView>
696 typename std::enable_if<(Kokkos::is_view<ExpView>::value && Kokkos::is_view<ImpView>::value)>::type
698 doReversePostsAndWaits (
const ExpView& exports,
700 const ImpView& imports)
702 doReversePosts (exports, numPackets, imports);
706 template <
class ExpView,
class ImpView>
707 typename std::enable_if<(Kokkos::is_view<ExpView>::value && Kokkos::is_view<ImpView>::value)>::type
709 doReversePostsAndWaits (
const ExpView& exports,
710 const Teuchos::ArrayView<const size_t>& numExportPacketsPerLID,
711 const ImpView& imports,
712 const Teuchos::ArrayView<const size_t>& numImportPacketsPerLID)
714 doReversePosts (exports, numExportPacketsPerLID, imports,
715 numImportPacketsPerLID);
719 template <
class ExpView,
class ImpView>
720 typename std::enable_if<(Kokkos::is_view<ExpView>::value && Kokkos::is_view<ImpView>::value)>::type
722 doReversePosts (
const ExpView &exports,
724 const ImpView &imports)
727 TEUCHOS_TEST_FOR_EXCEPTION(
728 ! plan_.getIndicesTo().is_null(), std::runtime_error,
729 "Tpetra::Distributor::doReversePosts(3 args): Can only do "
730 "reverse communication when original data are blocked by process.");
731 if (reverseDistributor_.is_null ()) {
732 createReverseDistributor ();
734 reverseDistributor_->doPosts (exports, numPackets, imports);
737 template <
class ExpView,
class ImpView>
738 typename std::enable_if<(Kokkos::is_view<ExpView>::value && Kokkos::is_view<ImpView>::value)>::type
740 doReversePosts (
const ExpView &exports,
741 const Teuchos::ArrayView<const size_t>& numExportPacketsPerLID,
742 const ImpView &imports,
743 const Teuchos::ArrayView<const size_t>& numImportPacketsPerLID)
746 TEUCHOS_TEST_FOR_EXCEPTION(
747 ! plan_.getIndicesTo().is_null(), std::runtime_error,
748 "Tpetra::Distributor::doReversePosts(3 args): Can only do "
749 "reverse communication when original data are blocked by process.");
750 if (reverseDistributor_.is_null ()) {
751 createReverseDistributor ();
753 reverseDistributor_->doPosts (exports, numExportPacketsPerLID,
754 imports, numImportPacketsPerLID);
757 template <
class OrdinalType>
759 computeSends(
const Teuchos::ArrayView<const OrdinalType>& importGIDs,
760 const Teuchos::ArrayView<const int>& importProcIDs,
761 Teuchos::Array<OrdinalType>& exportGIDs,
762 Teuchos::Array<int>& exportProcIDs)
771 using Teuchos::ArrayView;
773 using size_type =
typename ArrayView<const OrdinalType>::size_type;
774 const char errPrefix[] =
"Tpetra::Distributor::computeSends: ";
775 const char suffix[] =
776 " Please report this bug to the Tpetra developers.";
778 const int myRank = plan_.getComm()->getRank ();
780 TEUCHOS_TEST_FOR_EXCEPTION
781 (importGIDs.size () != importProcIDs.size (),
782 std::invalid_argument, errPrefix <<
"On Process " << myRank
783 <<
": importProcIDs.size()=" << importProcIDs.size()
784 <<
" != importGIDs.size()=" << importGIDs.size() <<
".");
786 const size_type numImports = importProcIDs.size();
787 Kokkos::View<size_t*, Kokkos::HostSpace> importObjs(
"importObjs", 2*numImports);
789 for (size_type i = 0; i < numImports; ++i) {
790 importObjs[2*i] =
static_cast<size_t>(importGIDs[i]);
791 importObjs[2*i+1] =
static_cast<size_t>(myRank);
800 const size_t numExportsAsSizeT =
801 tempPlan.createFromSends(importProcIDs);
802 const size_type numExports =
803 static_cast<size_type
>(numExportsAsSizeT);
804 TEUCHOS_TEST_FOR_EXCEPTION
805 (numExports < 0, std::logic_error, errPrefix <<
806 "tempPlan.createFromSends() returned numExports="
807 << numExportsAsSizeT <<
" as a size_t, which overflows to "
808 << numExports <<
" when cast to " <<
809 Teuchos::TypeNameTraits<size_type>::name () <<
"." << suffix);
810 TEUCHOS_TEST_FOR_EXCEPTION
811 (size_type(tempPlan.getTotalReceiveLength()) != numExports,
812 std::logic_error, errPrefix <<
"tempPlan.getTotalReceiveLength()="
813 << tempPlan.getTotalReceiveLength () <<
" != numExports="
814 << numExports <<
"." << suffix);
816 if (numExports > 0) {
817 exportGIDs.resize(numExports);
818 exportProcIDs.resize(numExports);
829 static_assert(
sizeof(
size_t) >=
sizeof(OrdinalType),
830 "Tpetra::Distributor::computeSends: "
831 "sizeof(size_t) < sizeof(OrdinalType).");
833 TEUCHOS_TEST_FOR_EXCEPTION
834 (tempPlan.getTotalReceiveLength () < size_t(numExports),
836 errPrefix <<
"tempPlan.getTotalReceiveLength()="
837 << tempPlan.getTotalReceiveLength() <<
" < numExports="
838 << numExports <<
"." << suffix);
840 Kokkos::View<size_t*, Kokkos::HostSpace> exportObjs(
"exportObjs", tempPlan.getTotalReceiveLength() * 2);
841 tempPlan.doPostsAndWaits(importObjs, 2, exportObjs);
844 for (size_type i = 0; i < numExports; ++i) {
845 exportGIDs[i] =
static_cast<OrdinalType
> (exportObjs[2*i]);
846 exportProcIDs[i] =
static_cast<int> (exportObjs[2*i+1]);
850 template <
class OrdinalType>
852 createFromRecvs (
const Teuchos::ArrayView<const OrdinalType> &remoteGIDs,
853 const Teuchos::ArrayView<const int> &remoteProcIDs,
854 Teuchos::Array<OrdinalType> &exportGIDs,
855 Teuchos::Array<int> &exportProcIDs)
858 const char errPrefix[] =
"Tpetra::Distributor::createFromRecvs: ";
859 const int myRank = plan_.getComm()->getRank();
861 std::unique_ptr<std::string> prefix;
863 prefix = createPrefix(
"createFromRecvs");
864 std::ostringstream os;
865 os << *prefix <<
"Start" << endl;
866 std::cerr << os.str();
871 using Teuchos::outArg;
872 using Teuchos::REDUCE_MAX;
873 using Teuchos::reduceAll;
877 (remoteGIDs.size () != remoteProcIDs.size ()) ? myRank : -1;
879 reduceAll(*plan_.getComm(), REDUCE_MAX, errProc, outArg(maxErrProc));
880 TEUCHOS_TEST_FOR_EXCEPTION
881 (maxErrProc != -1, std::runtime_error, errPrefix <<
"Lists "
882 "of remote IDs and remote process IDs must have the same "
883 "size on all participating processes. Maximum process ID "
884 "with error: " << maxErrProc <<
".");
889 TEUCHOS_TEST_FOR_EXCEPTION
890 (remoteGIDs.size() != remoteProcIDs.size(), std::runtime_error,
891 errPrefix <<
"On Process " << myRank <<
": "
892 "remoteGIDs.size()=" << remoteGIDs.size() <<
893 " != remoteProcIDs.size()=" << remoteProcIDs.size() <<
".");
896 computeSends(remoteGIDs, remoteProcIDs, exportGIDs, exportProcIDs);
898 plan_.createFromRecvs(remoteProcIDs);
901 std::ostringstream os;
902 os << *prefix <<
"Done" << endl;
903 std::cerr << os.str();
909 #endif // TPETRA_DISTRIBUTOR_HPP
const Details::DistributorPlan & getPlan() const
Get this Distributor's DistributorPlan.
size_t getNumReceives() const
The number of processes from which we will receive data.
std::string description() const
Return a one-line description of this object.
std::enable_if<(Kokkos::is_view< ExpView >::value &&Kokkos::is_view< ImpView >::value)>::type doReversePostsAndWaits(const ExpView &exports, size_t numPackets, const ImpView &imports)
Execute the reverse communication plan.
Teuchos::RCP< Distributor > getReverse(bool create=true) const
A reverse communication plan Distributor.
EDistributorHowInitialized
Enum indicating how and whether a Distributor was initialized.
Teuchos::RCP< const Teuchos::ParameterList > getValidParameters() const
List of valid Distributor parameters.
virtual ~Distributor()=default
Destructor (virtual for memory safety).
static bool debug()
Whether Tpetra is in debug mode.
std::enable_if<(Kokkos::is_view< ExpView >::value &&Kokkos::is_view< ImpView >::value)>::type doPosts(const ExpView &exports, size_t numPackets, const ImpView &imports)
Post the data for a forward plan, but do not execute the waits yet.
void swap(Distributor &rhs)
Swap the contents of rhs with those of *this.
Teuchos::ArrayView< const size_t > getLengthsFrom() const
Number of values this process will receive from each process.
Teuchos::ArrayView< const int > getProcsFrom() const
Ranks of the processes sending values to this process.
size_t createFromSends(const Teuchos::ArrayView< const int > &exportProcIDs)
Set up Distributor using list of process ranks to which this process will send.
Details::EDistributorHowInitialized howInitialized() const
Return an enum indicating whether and how a Distributor was initialized.
Teuchos::ArrayView< const int > getProcsTo() const
Ranks of the processes to which this process will send values.
void createFromSendsAndRecvs(const Teuchos::ArrayView< const int > &exportProcIDs, const Teuchos::ArrayView< const int > &remoteProcIDs)
Set up Distributor using list of process ranks to which to send, and list of process ranks from which...
bool hasSelfMessage() const
Whether the calling process will send or receive messages to itself.
std::enable_if<(Kokkos::is_view< ExpView >::value &&Kokkos::is_view< ImpView >::value)>::type doReversePosts(const ExpView &exports, size_t numPackets, const ImpView &imports)
Post the data for a reverse plan, but do not execute the waits yet.
Sets up and executes a communication plan for a Tpetra DistObject.
size_t getTotalReceiveLength() const
Total number of values this process will receive from other processes.
void setParameterList(const Teuchos::RCP< Teuchos::ParameterList > &plist)
Set Distributor parameters.
Teuchos::ArrayView< const size_t > getLengthsTo() const
Number of values this process will send to each process.
Teuchos::Array< std::string > distributorSendTypes()
Valid values for Distributor's "Send type" parameter.
std::enable_if<(Kokkos::is_view< ExpView >::value &&Kokkos::is_view< ImpView >::value)>::type doPostsAndWaits(const ExpView &exports, size_t numPackets, const ImpView &imports)
Execute the (forward) communication plan.
Stand-alone utility functions and macros.
size_t getNumSends() const
The number of processes to which we will send data.
void describe(Teuchos::FancyOStream &out, const Teuchos::EVerbosityLevel verbLevel=Teuchos::Describable::verbLevel_default) const
Describe this object in a human-readable way to the given output stream.
size_t getMaxSendLength() const
Maximum number of values this process will send to another single process.
void createFromRecvs(const Teuchos::ArrayView< const Ordinal > &remoteIDs, const Teuchos::ArrayView< const int > &remoteProcIDs, Teuchos::Array< Ordinal > &exportIDs, Teuchos::Array< int > &exportProcIDs)
Set up Distributor using list of process ranks from which to receive.
Distributor(const Teuchos::RCP< const Teuchos::Comm< int > > &comm)
Construct using the specified communicator and default parameters.
Declaration of Tpetra::Details::Behavior, a class that describes Tpetra's behavior.