Provides collectives that interoperate with the AM and task interfaces. More...

#include <worldgop.h>

Public Member Functions
	WorldGopInterface (World &world)

	~WorldGopInterface ()

bool	set_debug (bool value)
	Set debug flag to new value and return old value. More...

void	barrier ()
	Synchronizes all processes in communicator ... does NOT fence pending AM or tasks. More...

void	fence ()
	Synchronizes all processes in communicator AND globally ensures no pending AM or tasks. More...

void	broadcast (void *buf, size_t nbyte, ProcessID root, bool dowork=true)
	Broadcasts bytes from process root while still processing AM & tasks. More...

template<typename T >
void	broadcast (T *buf, size_t nelem, ProcessID root)
	Broadcasts typed contiguous data from process root while still processing AM & tasks. More...

template<typename T >
void	broadcast (T &t)
	Broadcast of a scalar from node 0 to all other nodes. More...

template<typename T >
void	broadcast (T &t, ProcessID root)
	Broadcast of a scalar from node root to all other nodes. More...

template<typename objT >
void	broadcast_serializable (objT &obj, ProcessID root)
	Broadcast a serializable object. More...

template<typename T , class opT >
void	reduce (T *buf, size_t nelem, opT op)
	Inplace global reduction (like MPI all_reduce) while still processing AM & tasks. More...

template<typename T >
void	sum (T *buf, size_t nelem)
	Inplace global sum while still processing AM & tasks. More...

template<typename T >
void	min (T *buf, size_t nelem)
	Inplace global min while still processing AM & tasks. More...

template<typename T >
void	max (T *buf, size_t nelem)
	Inplace global max while still processing AM & tasks. More...

template<typename T >
void	absmin (T *buf, size_t nelem)
	Inplace global absmin while still processing AM & tasks. More...

template<typename T >
void	absmax (T *buf, size_t nelem)
	Inplace global absmax while still processing AM & tasks. More...

template<typename T >
void	product (T *buf, size_t nelem)
	Inplace global product while still processing AM & tasks. More...

template<typename T >
void	bit_and (T *buf, size_t nelem)

template<typename T >
void	bit_or (T *buf, size_t nelem)

template<typename T >
void	bit_xor (T *buf, size_t nelem)

template<typename T >
void	logic_and (T *buf, size_t nelem)

template<typename T >
void	logic_or (T *buf, size_t nelem)

template<typename T >
void	sum (T &a)
	Global sum of a scalar while still processing AM & tasks. More...

template<typename T >
void	max (T &a)
	Global max of a scalar while still processing AM & tasks. More...

template<typename T >
void	min (T &a)
	Global min of a scalar while still processing AM & tasks. More...

template<typename T >
std::vector< T >	concat0 (const std::vector< T > &v, size_t bufsz=1024 *1024)
	Concatenate an STL vector of serializable stuff onto node 0. More...

template<typename keyT , typename valueT >
void	send (const ProcessID dest, const keyT &key, const valueT &value) const
	Send value to `dest`. More...

template<typename keyT , typename opT >
void	lazy_sync (const keyT &key, const opT &op) const
	Lazy sync. More...

template<typename keyT , typename opT >
void	lazy_sync (const keyT &key, const opT &op, const Group &group) const
	Group lazy sync. More...

template<typename keyT , typename valueT >
void	bcast (const keyT &key, Future< valueT > &value, const ProcessID root) const
	Broadcast. More...

template<typename keyT , typename valueT >
void	bcast (const keyT &key, Future< valueT > &value, const ProcessID group_root, const Group &group) const
	Group broadcast. More...

template<typename keyT , typename valueT , typename opT >
Future< typename detail::result_of< opT >::type >	reduce (const keyT &key, const valueT &value, const opT &op, const ProcessID root)
	Distributed reduce. More...

template<typename keyT , typename valueT , typename opT >
Future< typename detail::result_of< opT >::type >	reduce (const keyT &key, const valueT &value, const opT &op, const ProcessID group_root, const Group &group)
	Distributed group reduce. More...

template<typename keyT , typename valueT , typename opT >
Future< typename detail::result_of< opT >::type >	all_reduce (const keyT &key, const valueT &value, const opT &op)
	Distributed all reduce. More...

template<typename keyT , typename valueT , typename opT >
Future< typename detail::result_of< opT >::type >	all_reduce (const keyT &key, const valueT &value, const opT &op, const Group &group)
	Distributed, group all reduce. More...

Static Public Member Functions
template<typename valueT , typename keyT >
static Future< valueT >	recv (const ProcessID source, const keyT &key)
	Receive data from `source`. More...

Friends
class	detail::DeferredCleanup

Detailed Description

Provides collectives that interoperate with the AM and task interfaces.

If native AM interoperates with MPI we probably should map these to MPI.

Constructor & Destructor Documentation

madness::WorldGopInterface::WorldGopInterface ( World & world )

inline

madness::WorldGopInterface::~WorldGopInterface ( )

inline

References madness::detail::DeferredCleanup::destroy(), and madness::detail::DeferredCleanup::do_cleanup().

Member Function Documentation

template<typename T >

void madness::WorldGopInterface::absmax	(	T *	buf,
		size_t	nelem
	)

inline

Inplace global absmax while still processing AM & tasks.

template<typename T >

void madness::WorldGopInterface::absmin	(	T *	buf,
		size_t	nelem
	)

inline

Inplace global absmin while still processing AM & tasks.

template<typename keyT , typename valueT , typename opT >

Future<typename detail::result_of<opT>::type> madness::WorldGopInterface::all_reduce	(	const keyT &	key,
		const valueT &	value,
		const opT &	op
	)

inline

Distributed all reduce.

The reduce functor must have the following signature:

class ReduceFunc {
public:
    // Typedefs
    typedef ... result_type;
    tyepdef ... argument_type;
    // Constructors
    ReduceFunc(const ReduceFunc&);
    // Initialization operation, which returns a default result object
    result_type operator()() const;
    // Reduce two result objects
    void operator()(result_type&, const result_type&) const;
    // Reduce a result object and an argument object
    void operator()(result_type&, const argument_type&) const;
}; // class ReduceFunc

Template Parameters

keyT	The key type
valueT	The data type to be reduced
opT	The reduction operation type

Parameters

key	The key associated with this reduction
value	The local value to be reduced
op	The reduction operation to be applied to local and remote data
root	The process that will receive the result of the reduction

Returns: A future to the reduce value on the root process, otherwise an uninitialized future that may be ignored.

Note: It is the user's responsibility to ensure that key does not conflict with other calls to all_reduce. Keys may be reuse after the associated operation has finished.

References op(), and mpfr::root().

template<typename keyT , typename valueT , typename opT >

Future<typename detail::result_of<opT>::type> madness::WorldGopInterface::all_reduce	(	const keyT &	key,
		const valueT &	value,
		const opT &	op,
		const Group &	group
	)

inline

Distributed, group all reduce.

The reduce functor must have the following signature:

class ReduceFunc {
public:
    // Typedefs
    typedef ... result_type;
    typedef ... argument_type;
    // Constructors
    ReduceFunc(const ReduceFunc&);
    // Initialization operation, which returns a default result object
    result_type operator()() const;
    // Reduce two result objects
    void operator()(result_type&, const result_type&) const;
    // Reduce a result object and an argument object
    void operator()(result_type&, const argument_type&) const;
}; // class ReduceFunc

Template Parameters

keyT	The key type
valueT	The data type to be reduced
opT	The reduction operation type

Parameters

key	The key associated with this reduction
value	The local value to be reduced
op	The reduction operation to be applied to local and remote data
group_root	The group process that will receive the result of the reduction
group	The group that will preform the reduction

Returns: A future to the reduce value on the root process, otherwise an uninitialized future that may be ignored

Exceptions

madness::Exception	When `group` is empty
madness::Exception	When `group` is not registered
madness::Exception	When the world id of `group` is not equal to that of the world used to construct this object
madness::Exception	When this process is not in the group

Note: It is the user's responsibility to ensure that key does not conflict with other calls to reduce. Keys may be reuse after the associated operation has finished.

References madness::Group::empty(), madness::Group::get_world(), madness::World::id(), madness::Group::make_tree(), op(), madness::Group::rank(), and madness::Group::size().

void madness::WorldGopInterface::barrier ( )

inline

Synchronizes all processes in communicator ... does NOT fence pending AM or tasks.

References madness::error(), and sum().

Referenced by madness::plotvtk_data(), and madness::World::World().

template<typename keyT , typename valueT >

void madness::WorldGopInterface::bcast	(	const keyT &	key,
		Future< valueT > &	value,
		const ProcessID	root
	)		const

inline

Broadcast.

Broadcast data from the root process to all processes. The input/ output data is held by value.

Parameters

[in]	key	The key associated with this broadcast
[in,out]	value	On the `root` process, this is used as the input data that will be broadcast to all other processes. On other processes it is used as the output to the broadcast.
	root	The process that owns the data to be broadcast

Exceptions

madness::Exception	When `root` is less than 0 or greater than or equal to the world size.
madness::Exception	When `value` has been set, except on the `root` process.

Note: It is the user's responsibility to ensure that key does not conflict with other calls to bcast. Keys may be reuse after the associated operation has finished.

References madness::Future< T >::probe(), and mpfr::root().

template<typename keyT , typename valueT >

void madness::WorldGopInterface::bcast	(	const keyT &	key,
		Future< valueT > &	value,
		const ProcessID	group_root,
		const Group &	group
	)		const

inline

Group broadcast.

Broadcast data from the group_root process to all processes in group. The input/output data is held by value.

Parameters

[in]	key	The key associated with this broadcast
[in,out]	value	On the `group_root` process, this is used as the input data that will be broadcast to all other processes in the group. On other processes it is used as the output to the broadcast
	group_root	The process in `group` that owns the data to be broadcast
	group	The process group where value will be broadcast

Exceptions

madness::Exception	When `group` is empty
madness::Exception	When `group` is not registered
madness::Exception	When the world id of `group` is not equal to that of the world used to construct this object
madness::Exception	When this process is not in the group
madness::Exception	When `group_root` is less than 0 or greater than or equal to `group` size
madness::Exception	When `data` has been set except on the `root` process

Note: It is the user's responsibility to ensure that key does not conflict with other calls to bcast. Keys may be reuse after the associated operation has finished.

References madness::Group::empty(), madness::Group::get_world(), madness::World::id(), madness::Future< T >::probe(), madness::Group::rank(), and madness::Group::size().

template<typename T >

void madness::WorldGopInterface::bit_and	(	T *	buf,
		size_t	nelem
	)

inline

template<typename T >

void madness::WorldGopInterface::bit_or	(	T *	buf,
		size_t	nelem
	)

inline

template<typename T >

void madness::WorldGopInterface::bit_xor	(	T *	buf,
		size_t	nelem
	)

inline

void madness::WorldGopInterface::broadcast	(	void *	buf,
		size_t	nbyte,
		ProcessID	root,
		bool	dowork = `true`
	)

Broadcasts bytes from process root while still processing AM & tasks.

Optimizations can be added for long messages

References madness::World::await(), SafeMPI::Intracomm::binary_tree_info(), madness::WorldMpiInterface::Irecv(), madness::WorldMpiInterface::Isend(), madness::World::mpi, MPI_BYTE, and SafeMPI::Intracomm::unique_tag().

Referenced by broadcast(), broadcast_serializable(), SCF::diag_fock_matrix(), doit(), madness::archive::BaseParallelArchive< BinaryFstreamInputArchive >::exists(), fence(), madness::SCF::get_fock_transformation(), madness::load_coeffs(), madness::load_quadrature(), SCF::localize_boys(), main(), madness::archive::BaseParallelArchive< BinaryFstreamInputArchive >::open(), propagate(), reduce(), madness::SCF::update_subspace(), and madness::World::World().

template<typename T >

void madness::WorldGopInterface::broadcast	(	T *	buf,
		size_t	nelem,
		ProcessID	root
	)

inline

Broadcasts typed contiguous data from process root while still processing AM & tasks.

Optimizations can be added for long messages

References broadcast(), and std::tr1::T().

template<typename T >

void madness::WorldGopInterface::broadcast ( T & t )

inline

Broadcast of a scalar from node 0 to all other nodes.

References broadcast().

template<typename T >

void madness::WorldGopInterface::broadcast	(	T &	t,
		ProcessID	root
	)

inline

Broadcast of a scalar from node root to all other nodes.

References broadcast().

template<typename objT >

void madness::WorldGopInterface::broadcast_serializable	(	objT &	obj,
		ProcessID	root
	)

inline

Broadcast a serializable object.

Current dumb version assumes object fits in 1MB ... you are free to add intelligence.

References broadcast(), mpfr::root(), and madness::archive::BufferOutputArchive::size().

Referenced by madness::archive::BaseParallelArchive< BinaryFstreamInputArchive >::broadcast(), doit(), madness::Solver< T, NDIM >::init(), madness::LoadBalanceDeux< NDIM >::load_balance(), main(), madness::MP2::MP2(), madness::SCF::SCF(), madness::Subspace< T, NDIM >::update_subspace(), and madness::SCF::update_subspace().

template<typename T >

std::vector<T> madness::WorldGopInterface::concat0	(	const std::vector< T > &	v,
		size_t	bufsz = `1024*1024`
	)

inline

Concatenate an STL vector of serializable stuff onto node 0.

References madness::World::await(), MPI_BYTE, and madness::archive::BufferOutputArchive::size().

Referenced by madness::LoadBalanceDeux< NDIM >::load_balance().

void madness::WorldGopInterface::fence ( )

Synchronizes all processes in communicator AND globally ensures no pending AM or tasks.

Runs Dykstra-like termination algorithm on binary tree by locally ensuring ntask=0 and all am sent and processed, and then participating in a global sum of nsent and nrecv. Then globally checks that nsent=nrecv and that both are constant over two traversals. We are then we are sure that all tasks and AM are processed and there no AM in flight.

References madness::World::am, madness::World::await(), SafeMPI::Intracomm::binary_tree_info(), broadcast(), madness::detail::DeferredCleanup::do_cleanup(), madness::WorldTaskQueue::fence(), madness::WorldAmInterface::free_managed_buffers(), madness::WorldMpiInterface::Irecv(), madness::WorldMpiInterface::Isend(), madness::World::mpi, MPI_BYTE, PROFILE_MEMBER_FUNC, madness::WorldTaskQueue::size(), sum(), madness::World::taskq, and SafeMPI::Intracomm::unique_tag().

template<typename keyT , typename opT >

void madness::WorldGopInterface::lazy_sync	(	const keyT &	key,
		const opT &	op
	)		const

inline

Lazy sync.

Lazy sync functions are asynchronous barriers with a nullary functor that is called after all processes have called it with the same key. You can think of lazy_sync as an asynchronous barrier. The lazy_sync functor must have the following signature:

class SyncFunc {
public:
    // typedefs
    typedef void result_type;
    // Constructors
    SyncFunc(const SyncFunc&);
    // The function that performs the sync operation
    void operator()();
}; // class SyncFunc

Template Parameters

keyT	The key type
opT	The operation type

Parameters

key	The sync key
op	The sync operation to be executed on this process

Note: It is the user's responsibility to ensure that key does not conflict with other calls to lazy_sync. Keys may be reuse after the associated operation has finished.

References madness::TaskAttributes::hipri(), and op().

template<typename keyT , typename opT >

void madness::WorldGopInterface::lazy_sync	(	const keyT &	key,
		const opT &	op,
		const Group &	group
	)		const

inline

Group lazy sync.

Lazy sync functions are asynchronous barriers with a nullary functor that is called after all processes in the group have called it with the same key. You can think of lazy_sync as an asynchronous barrier. The op functor must have the following signature:

class SyncFunc {
public:
    // typedefs
    typedef void result_type;
    // Constructors
    SyncFunc(const SyncFunc&);
    // The function that performs the sync operation
    void operator()();
}; // class SyncFunc

Template Parameters

keyT	The key type
opT	The operation type

Parameters

key	The sync key
op	The sync operation to be executed on this process

Note: It is the user's responsibility to ensure that key does not conflict with other calls to lazy_sync. Keys may be reuse after the associated operation has finished.

References madness::Group::empty(), madness::Group::get_world(), madness::TaskAttributes::hipri(), madness::World::id(), madness::Group::make_tree(), op(), and madness::Group::size().

template<typename T >

void madness::WorldGopInterface::logic_and	(	T *	buf,
		size_t	nelem
	)

inline

template<typename T >

void madness::WorldGopInterface::logic_or	(	T *	buf,
		size_t	nelem
	)

inline

template<typename T >

void madness::WorldGopInterface::max	(	T *	buf,
		size_t	nelem
	)

inline

Inplace global max while still processing AM & tasks.

Referenced by max(), and madness::print_stats().

template<typename T >

void madness::WorldGopInterface::max ( T & a )

inline

Global max of a scalar while still processing AM & tasks.

References max().

template<typename T >

void madness::WorldGopInterface::min	(	T *	buf,
		size_t	nelem
	)

inline

Inplace global min while still processing AM & tasks.

Referenced by min(), and madness::print_stats().

template<typename T >

void madness::WorldGopInterface::min ( T & a )

inline

Global min of a scalar while still processing AM & tasks.

References min().

template<typename T >

void madness::WorldGopInterface::product	(	T *	buf,
		size_t	nelem
	)

inline

Inplace global product while still processing AM & tasks.

template<typename valueT , typename keyT >

static Future<valueT> madness::WorldGopInterface::recv	(	const ProcessID	source,
		const keyT &	key
	)

inlinestatic

Receive data from source.

Template Parameters

valueT	The data type stored in cache
keyT	The key type

Parameters

source	The process that is sending the data to this process
key	The key associated with the received data

Returns: A future that will be set with the received data

Note: It is the user's responsibility to ensure that key does not conflict with other calls to recv. Keys may be reuse after the associated operation has finished.

template<typename T , class opT >

void madness::WorldGopInterface::reduce	(	T *	buf,
		size_t	nelem,
		opT	op
	)

inline

Inplace global reduction (like MPI all_reduce) while still processing AM & tasks.

Optimizations can be added for long messages and to reduce the memory footprint

References madness::World::await(), broadcast(), MPI_BYTE, op(), and std::tr1::T().

template<typename keyT , typename valueT , typename opT >

Future<typename detail::result_of<opT>::type> madness::WorldGopInterface::reduce	(	const keyT &	key,
		const valueT &	value,
		const opT &	op,
		const ProcessID	root
	)

inline

Distributed reduce.

The reduce functor must have the following signature:

class ReduceFunc {
public:
    // Typedefs
    typedef ... result_type;
    tyepdef ... argument_type;
    // Constructors
    ReduceFunc(const ReduceFunc&);
    // Initialization operation, which returns a default result object
    result_type operator()() const;
    // Reduce two result objects
    void operator()(result_type&, const result_type&) const;
    // Reduce a result object and an argument object
    void operator()(result_type&, const argument_type&) const;
}; // class ReduceFunc

Template Parameters

keyT	The key type
valueT	The data type to be reduced
opT	The reduction operation type

Parameters

key	The key associated with this reduction
value	The local value to be reduced
op	The reduction operation to be applied to local and remote data
root	The process that will receive the result of the reduction

Returns: A future to the reduce value on the root process, otherwise an uninitialized future that may be ignored.

Note: It is the user's responsibility to ensure that key does not conflict with other calls to reduce. Keys may be reuse after the associated operation has finished.

References op(), and mpfr::root().

template<typename keyT , typename valueT , typename opT >

Future<typename detail::result_of<opT>::type> madness::WorldGopInterface::reduce	(	const keyT &	key,
		const valueT &	value,
		const opT &	op,
		const ProcessID	group_root,
		const Group &	group
	)

inline

Distributed group reduce.

The reduce functor must have the following signature:

class ReduceFunc {
public:
    // Typedefs
    typedef ... result_type;
    tyepdef ... argument_type;
    // Constructors
    ReduceFunc(const ReduceFunc&);
    // Initialization operation, which returns a default result object
    result_type operator()() const;
    // Reduce two result objects
    void operator()(result_type&, const result_type&) const;
    // Reduce a result object and an argument object
    void operator()(result_type&, const argument_type&) const;
}; // class ReduceFunc

Template Parameters

keyT	The key type
valueT	The data type to be reduced
opT	The reduction operation type

Parameters

key	The key associated with this reduction
value	The local value to be reduced
op	The reduction operation to be applied to local and remote data
group_root	The group process that will receive the result of the reduction
group	The group that will preform the reduction

Returns: A future to the reduce value on the root process, otherwise an uninitialized future that may be ignored.

Exceptions

madness::Exception	When `group` is empty
madness::Exception	When `group` is not registered
madness::Exception	When the world id of `group` is not equal to that of the world used to construct this object
madness::Exception	When this process is not in the group
madness::Exception	When `group_root` is less than zero or greater than or equal to `group` size.

Note: It is the user's responsibility to ensure that key does not conflict with other calls to reduce. Keys may be reuse after the associated operation has finished.

References madness::Group::empty(), madness::Group::get_world(), madness::World::id(), madness::Group::make_tree(), op(), and madness::Group::size().

template<typename keyT , typename valueT >

void madness::WorldGopInterface::send	(	const ProcessID	dest,
		const keyT &	key,
		const valueT &	value
	)		const

inline

Send value to dest.

Template Parameters

keyT	The key type
valueT	The value type (this may be a `Future` type)

Parameters

dest	The process where the data will be sent
key	The key that is associated with the data
value	The data to be sent to `dest`

Note: It is the user's responsibility to ensure that key does not conflict with other calls to send. Keys may be reuse after the associated operation has finished.

bool madness::WorldGopInterface::set_debug ( bool value )

inline

Set debug flag to new value and return old value.

template<typename T >

void madness::WorldGopInterface::sum	(	T *	buf,
		size_t	nelem
	)

inline

Inplace global sum while still processing AM & tasks.

Referenced by barrier(), madness::DistributedMatrix< double >::copy_to_replicated(), madness::DistributedMatrix< double >::copy_to_replicated_patch(), CubicInterpolationTable< complexd >::CubicInterpolationTable(), fence(), madness::inner(), line_plot(), madness::matrix_inner(), madness::norm2(), madness::norm2s(), madness::print_stats(), projectL(), sum(), madness::Subspace< T, NDIM >::update_subspace(), and madness::SCF::update_subspace().

template<typename T >

void madness::WorldGopInterface::sum ( T & a )

inline

Global sum of a scalar while still processing AM & tasks.

References sum().

Friends And Related Function Documentation

friend class detail::DeferredCleanup

friend

The documentation for this class was generated from the following files:

Public Member Functions

Static Public Member Functions

Friends

Detailed Description

Constructor & Destructor Documentation

Member Function Documentation

Friends And Related Function Documentation