libglobalarrays-dev 5.4~beta~r10636+dfsg-5 / usr / include / GAServices.h

#ifndef _GA_SERVICES_H
#define _GA_SERVICES_H

namespace GA {

class GlobalArray;

/**              
 * Global Arrays Services class.
 * 
 * @author Manoj Kumar Krishnan, PNNL. 
 * @deprecated Please use the same functions found within the GA namespace.
 * 
 * Collecting the global information: who am I, and how many processors 
 * are being used. Initialize the communication library (either MPI or 
 * TCSMSG) and Global array. Allocate momory to be used by GA by calling MA
 * and create global arrays.
 */
class GAServices {

public:
  /**  
   * Null-constructor. The component won't really be 'alive'
   * much at all until after setServices is called on it.
   */
  GAServices();
  
  /** Destructor. */
  ~GAServices();
  
  /**  
   * Creates an ndim-dimensional array using the regular distribution model 
   * and returns integer handle representing the array. 
   *
   * The array can be distributed evenly or not. The control over the 
   * distribution is accomplished by specifying chunk (block) size for all or 
   * some of array dimensions.
   *
   * For example, for a 2-dimensional array, setting chunk[0]=dim[0] gives 
   * distribution by vertical strips (chunk[0]*dims[0]); 
   * setting chunk[1]=dim[1] gives distribution by horizontal strips 
   * (chunk[1]*dims[1]). Actual chunks will be modified so that they are at 
   * least the size of the minimum and each process has either zero or one 
   * chunk. Specifying chunk[i] as <1 will cause that dimension to be 
   * distributed evenly. 
   *
   * As a convenience, when chunk is specified as NULL, the entire array is 
   * distributed evenly.
   *
   * This is a collective operation. 
   *
   * @param[in] type        data type(MT_F_DBL,MT_F_INT,MT_F_DCPL)
   * @param[in] ndim        number of array dimensions
   * @param[in] dims[ndim]  array of dimensions
   * @param[in] arrayname   a unique character string
   * @param[in] chunk[ndim] array of chunks, each element specifies 
   *                        minimum size that given dimensions should be
   *                        chunked up into
   *
   * @return pointer to GlobalArray object created; NULL if it fails
   */
  GlobalArray * createGA(int type, int ndim, int dims[], char *arrayname, 
          int chunk[]);
  
  /**
   * Creates an array by following the user-specified distribution and 
   * returns integer handle representing the array. 
   *
   * The distribution is specified as a Cartesian product of distributions 
   * for each dimension. The array indices start at 0. For example, the 
   * following figure demonstrates distribution of a 2-dimensional array 8x10 
   * on 6 (or more) processors. nblock[2]={3,2}, the size of map array is s=5 
   * and array map contains the following elements map={0,2,6, 0, 5}. The 
   * distribution is nonuniform because, P1 and P4 get 20 elements each and 
   * processors P0,P2,P3, and P5 only 10 elements each. 
   *        
   * <TABLE>
   * <TR> <TD>5</TD>  <TD>5</TD>  </TR>
   * <TR> <TD>P0</TD> <TD>P3</TD> <TD>2</TD> </TR>
   * <TR> <TD>P1</TD> <TD>P4</TD> <TD>4</TD> </TR>
   * <TR> <TD>P2</TD> <TD>P5</TD> <TD>2</TD> </TR>
   * </TABLE>
   *
   * This is a collective operation. 
   *
   * @param[in] arrayname   a unique character string
   * @param[in] type        MA data type (MT_F_DBL,MT_F_INT,MT_F_DCPL)
   * @param[in] ndim        number of array dimensions
   * @param[in] dims        array of dimension values
   * @param[in] block       [ndim] no. of blocks each dimension is divided into
   * @param[in] maps        [s] starting index for for each block;
   *                        the size s is a sum all elements of nblock array
   *
   * @return pointer to GlobalArray object created; NULL if it fails
   */
  GlobalArray * createGA(int type, int ndim, int dims[], char *arrayname, 
          int block[], int maps[]);
  
  /**
   * Creates a new array by applying all the properties of another existing 
   * array.
   *
   * This is a collective operation. 
   *
   * @param[in] arrayname a character string
   * @param[in] g_b       integer handle for reference array
   *
   * @return pointer to GlobalArray object created; NULL if it fails
   */
  GlobalArray * createGA(const GlobalArray *g_b, char *arrayname);
  
  /**
   * Creates a new array by applying all the properties of another existing 
   * array.
   *
   * This is a collective operation. 
   *
   * @param[in] g_b integer handle for reference array
   *
   * @return pointer to GlobalArray object created; NULL if it fails
   */
  GlobalArray * createGA(const GlobalArray &g_b);
  
  /**
   * Creates a 10x10 global array of type "double"(default).
   *
   * This is a collective operation. 
   *
   * @return pointer to GlobalArray object created; NULL if it fails
   */
  GlobalArray * createGA();  
  
  /**
   * Creates an ndim-dimensional array with a layer of ghost cells around 
   * the visible data on each processor using the regular distribution 
   * model and returns an integer handle representing the array. 
   * The array can be distributed evenly or not evenly. The control over 
   * the distribution is accomplished by specifying chunk (block) size for 
   * all or some of the array dimensions. For example, for a 2-dimensional 
   * array, setting chunk(1)=dim(1) gives distribution by vertical strips 
   * (chunk(1)*dims(1)); setting chunk(2)=dim(2) gives distribution by 
   * horizontal strips (chunk(2)*dims(2)). Actual chunks will be modified 
   * so that they are at least the size of the minimum and each process 
   * has either zero or one chunk. Specifying chunk(i) as <1 will cause
   * that dimension (i-th) to be distributed evenly. The  width of the 
   * ghost cell layer in each dimension is specified using the array 
   * width().  The local data of the global array residing on each 
   * processor will have a layer width[n] ghosts cells wide on either 
   * side of the visible data along the dimension n. 
   *
   * This is a collective operation. 
   * 
   * @param[in] array_name a unique character string
   * @param[in] type       data type (MT_DBL,MT_INT,MT_DCPL)
   * @param[in] ndim       number of array dimensions
   * @param[in] dims       [ndim] array of dimensions
   * @param[in] width      [ndim] array of ghost cell widths
   * @param[in] chunk      [ndim] array of chunks, each element specifies
   *                       minimum size that given dimensions should be
   *                       chunked up into
   *
   * @returns pointer to GlobalArray object created; NULL if it fails
   */
  GlobalArray * createGA_Ghosts(int type, int ndim, int dims[], 
          int width[], char *array_name, int chunk[]);
  
  /**
   * Creates an array with ghost cells by following the user-specified 
   * distribution and returns integer handle representing the array. 
   * The distribution is specified as a Cartesian product of distributions 
   * for each dimension. For example, the following figure demonstrates 
   * distribution of a 2-dimensional array 8x10 on 6 (or more) processors. 
   * nblock(2)={3,2}, the size of map array is s=5 and array map contains 
   * the following elements map={1,3,7, 1, 6}. The distribution is 
   * nonuniform because, P1 and P4 get 20 elements each and processors 
   * P0,P2,P3, and P5 only 10 elements each. 
   *
   * <TABLE>
   * <TR> <TD>5</TD>  <TD>5</TD>  </TR>
   * <TR> <TD>P0</TD> <TD>P3</TD> <TD>2</TD> </TR>
   * <TR> <TD>P1</TD> <TD>P4</TD> <TD>4</TD> </TR>
   * <TR> <TD>P2</TD> <TD>P5</TD> <TD>2</TD> </TR>
   * </TABLE>
   *
   * The array width[] is used to control the width of the ghost cell 
   * boundary around the visible data on each processor. The local data 
   * of the global array residing on each processor will have a layer 
   * width[n] ghosts cells wide on either side of the visible data along 
   * the dimension n. 
   *
   * This is a collective operation. 
   *
   * @param[in] array_name a unique character string
   * @param[in] type       data type (MT_DBL,MT_INT,MT_DCPL)
   * @param[in] ndim       number of array dimensions
   * @param[in] dims       [ndim] array of dimensions
   * @param[in] width      [ndim] array of ghost cell widths
   * @param[in] nblock     [ndim] no. of blocks each dimension is divided into
   * @param[in] map        [s] starting index for for each block;
   *                       the size s is a sum of all elements of nblock array
   *
   * @return pointer to GlobalArray object created; NULL if it fails
   */
  GlobalArray * createGA_Ghosts(int type, int ndim, int dims[], 
          int width[], char *array_name, int map[], 
          int nblock[]);
  
  /**
   * Broadcast from process root to all other processes a message of 
   * length lenbuf. This is operation is provided only for convenience 
   * purposes: it is available regardless of the message-passing library 
   * that GA is running with. 
   *
   * This is a collective operation. 
   *
   * @param[in]     lenbuf length of buffer
   * @param[in,out] buf    [lenbuf] data
   * @param[in]     root   root process
   */
  void brdcst(void *buf, int lenbuf, int root);
  
  /**
   * Returns the current value of the internal debug flag.
   *
   * This is a local operation.
   *
   * @return 0 if the debug flag is false, 1 if it is true.
   */
  int getDebug();
  
  /**
   * This functions returns the total number of nodes that the program is 
   * running on.
   *
   * On SMP architectures, this will be less than or equal to the total
   * number of processors. 
   *
   * This is a  local operation. 
   *
   * @return the number of nodes the program is running on
   */
  int clusterNnodes();
  
  /**  
   * This function returns the node ID of the process.
   *
   * On SMP architectures with more than one processor per node, several
   * processes may return the same node id. 
   *
   * This is a  local operation. 
   *
   * @return the node ID of the process
   */
  int clusterNodeid();
  
  /**  
   * This function returns the cluster node ID of the specified process.
   *
   * On SMP architectures with more than one processor per node, several
   * processes may return the same node id. 
   *
   * This is a  local operation. 
   *
   * @return the cluster node ID of the specified process
   */
  int clusterProcNodeid(int iproc);
  
  /**
   * This function returns the number of processors available on node inode. 
   *
   * This is a  local operation. 
   *
   * @param[in] inode
   *
   * @return the number of processors available on the given node
   */
  int clusterNprocs(int inode);
  
  /**
   * This function returns the processor id associated with node inode and 
   * the local processor id iproc.
   *
   * If node inode has N processors, then the value of iproc lies between
   * 0 and N-1. 
   *
   * This is a  local operation. 
   *
   * @param[in] inode
   * @param[in] iproc
   *
   * @return the processor ID associated with the given node and local processor
   * ID
   */
  int clusterProcid(int inode, int iproc);
  
  /**
   * Creates a set containing the number of mutexes.
   *
   * Mutex is a simple synchronization object used to protect Critical
   * Sections. Only one set of mutexes can exist at a time. Array of mutexes
   * can be created and destroyed as many times as needed. 
   * Mutexes are numbered: 0, ..., number -1.
   *
   * This is a collective operation. 
   *
   * @param[in] number of mutexes in mutex array
   *
   * @return 0 if the operation succeeded or 1 when failed. 
   */
  int createMutexes(int number);

  /**
   * Remove a user defined data type from GA
   *
   * @param[in] type - user defined data type
   *
   * @return 0 is operation is successful
   *         -2 if type not registered
   *         -1 if type reserved
   */
  int deregisterType(int type);
  
  /** 
   * Destroys the set of mutexes created with ga_create_mutexes.
   *
   * This is a collective operation. 
   *
   * @return 0 if the operation succeeded or 1 when failed. 
   */
  int destroyMutexes();
  
  /**
   * Double Global OPeration. 
   *
   * X(1:N) is a vector present on each process. DGOP 'sums' elements of 
   * X accross all nodes using the commutative operator OP. The result is 
   * broadcast to all nodes. Supported operations include '+', '*', 'max', 
   * 'min', 'absmax', 'absmin'. The use of lowerecase for operators is 
   * necessary. This is operation is provided only for convenience purposes: 
   * it is available regardless of the message-passing library that GA is 
   * running with.
   *
   * This is a collective operation. 
   *
   * @param[in]     n  number of elements
   * @param[in,out] x  [n] array of elements
   * @param[in]     op operator
   */
  void dgop(double x[], int n, char *op);
  
  /**
   * Creates a new array by applying all the properties of another existing 
   * array.
   *
   * This is a collective operation. 
   *
   * @param[in] array_name a character string
   * @param[in] g_a        integer handle for reference array
   *
   * @return array handle; a non-zero array handle means the call was succesful. 
   */
  int duplicate(int g_a, char* array_name);
  
  /**
   * To be called in case of an error.
   *
   * Print an error message and an integer value that represents error code.
   * Releases some system resources. 
   * This is the required way of aborting the program execution. 
   *
   * This operation is local. 
   *
   * @param[in] message string to print
   * @param[in] code    code to print
   */
  void error(const char *message, int code);
  
  /**
   * Blocks the calling process until all the data transfers corresponding to 
   * GA operations called after ga_init_fence complete.
   *
   * For example, since ga_put might return before the data reaches the final
   * destination, ga_init_fence and ga_fence allow process to wait until the
   * data tranfer is fully completed: 
   *
   * @code
   *   ga_init_fence();
   *   ga_put(g_a, ...);
   *   ga_fence();
   * @endcode
   * 
   * ga_fence must be called after ga_init_fence. A barrier, ga_sync, assures 
   * completion of all data transfers and implicitly cancels all outstanding
   * ga_init_fence calls. ga_init_fence and ga_fence must be used in pairs, 
   * multiple calls to ga_fence require the same number of corresponding
   * ga_init_fence calls. ga_init_fence/ga_fence pairs can be nested. 
   * 
   * ga_fence works for multiple GA operations. For example: 
   * 
   * @code
   *   ga_init_fence();
   *   ga_put(g_a, ...);
   *   ga_scatter(g_a, ...);
   *   ga_put(g_b, ...);
   *   ga_fence();
   * @endcode
   *
   * The calling process will be blocked until data movements initiated by 
   * two calls to ga_put and one ga_scatter complete. 
   */
  void fence();

  /**
   * Integer Global OPeration.
   *
   * The integer version of ga_dgop described above, also include the bitwise OR
   * operation.  This is operation is provided only for convenience purposes: it
   * is available regardless of the message-passing library that GA is running
   * with.
   *
   * This is a collective operation. 
   *
   * @param[in]     n  number of elements
   * @param[in,out] x  [n] array of elements
   * @param[in]     op operator
   */
  void gop(int x[], int n, char *op);
  
  /**
   * Long Global OPeration. 
   *
   * X(1:N) is a vector present on each process. LGOP 'sums' elements of 
   * X accross all nodes using the commutative operator OP. The result is 
   * broadcast to all nodes. Supported operations include '+', '*', 'max', 
   * 'min', 'absmax', 'absmin'. The use of lowerecase for operators is 
   * necessary. This is operation is provided only for convenience purposes: 
   * it is available regardless of the message-passing library that GA is 
   * running with.
   *
   * This is a collective operation. 
   *
   * @param[in]     n  number of elements
   * @param[in,out] x  [n] array of elements
   * @param[in]     op operator
   */
  void gop(long x[], int n, char *op);
  
  /**
   * Float Global OPeration. 
   *
   * X(1:N) is a vector present on each process. FGOP 'sums' elements of 
   * X accross all nodes using the commutative operator OP. The result is 
   * broadcast to all nodes. Supported operations include '+', '*', 'max', 
   * 'min', 'absmax', 'absmin'. The use of lowerecase for operators is 
   * necessary. This is operation is provided only for convenience purposes: 
   * it is available regardless of the message-passing library that GA is 
   * running with.
   *
   * This is a collective operation. 
   *
   * @param[in]     n  number of elements
   * @param[in,out] x  [n] array of elements
   * @param[in]     op operator
   */
  void gop(float x[], int n, char *op);
  
  /**
   * Double Global OPeration. 
   *
   * X(1:N) is a vector present on each process. DGOP 'sums' elements of 
   * X accross all nodes using the commutative operator OP. The result is 
   * broadcast to all nodes. Supported operations include '+', '*', 'max', 
   * 'min', 'absmax', 'absmin'. The use of lowerecase for operators is 
   * necessary. This is operation is provided only for convenience purposes: 
   * it is available regardless of the message-passing library that GA is 
   * running with.
   *
   * This is a collective operation. 
   *
   * @param[in]     n  number of elements
   * @param[in,out] x  [n] array of elements
   * @param[in]     op operator
   */
  void gop(double x[], int n, char *op);
  
  /**
   * Integer Global OPeration.
   *
   * The integer (more precisely long) version of ga_dgop described above,
   * also include the bitwise OR operation. 
   * This is operation is provided only for convenience purposes: it is 
   * available regardless of the message-passing library that GA is running 
   * with.
   *
   * This is a collective operation. 
   *
   * @param[in]     n  number of elements
   * @param[in,out] x  [n] array of elements
   * @param[in]     op operator
   */
  void igop(int x[], int n, char *op);
  
  /**
   * Initializes tracing of completion status of data movement operations. 
   *
   * This operation is local. 
   */
  void initFence();
  
  /**
   * Returns amount of memory (in bytes) used in the allocated global 
   * arrays on the calling processor.
   *
   * This operation is local. 
   *
   * @return amount of memory (in bytes) used in the allocated global arrays on
   *         the calling processor
   */
  size_t inquireMemory();
  
  /**
   * Long Global OPeration. 
   *
   * X(1:N) is a vector present on each process. LGOP 'sums' elements of 
   * X accross all nodes using the commutative operator OP. The result is 
   * broadcast to all nodes. Supported operations include '+', '*', 'max', 
   * 'min', 'absmax', 'absmin'. The use of lowerecase for operators is 
   * necessary. This is operation is provided only for convenience purposes: 
   * it is available regardless of the message-passing library that GA is 
   * running with.
   *
   * This is a collective operation. 
   *
   * @param[in]     n  number of elements
   * @param[in,out] x  [n] array of elements
   * @param[in]     op operator
   */
  void lgop(long x[], int n, char *op);
  
  /**
   * Locks a mutex object identified by the mutex number. It is a fatal 
   * error for a process to attempt to lock a mutex which was already 
   * locked by this process. 
   *
   * @param[in] mutex object id
   */
  void lock(int mutex);
  
  /**
   * Mask the intrinsic sync operations during collective calls.
   *
   * GA Collective calls has Sync calls at the begining and ending of
   * of the call. Sometimes there may be some redundacy in sync calls, which
   * can be avoided by masking the sync operations. 
   *
   * Setting the parameters as zero will mask (disable) the call. Any non-zero 
   * value will enable the call. Initially these params are set to non-zero 
   * value.
   *
   * @param[in] first masks the sync at the begining of the collective call.
   * @param[in] last  masks the sync at the end of the collective call.
   */
  void maskSync(int first, int last);
  
  /**
   * If GA_uses_ma returns true, then GA_Memory_avail returns the 
   * lesser of the amount available under the GA limit and the amount 
   * available from MA (according to ma_inquire_avail operation). 
   * If no GA limit has been set, it returns what MA says is available. 
   * If ( ! GA_Uses_ma() && ! GA_Memory_limited() ) returns < 0, indicating 
   * that the bound on currently available memory cannot be determined. 
   *
   * This operation is local. 
   *
   * @return amount of memory (in bytes) left for allocation of new 
   *         global arrays on the calling processor. 
   *
   */
  int memoryAvailable() ;
  
  /**
   * Indicates if limit is set on memory usage in Global Arrays on the 
   * calling processor.
   * 
   * This operation is local. 
   *
   * @return 1 means "yes", "0" means "no".
   */
  int memoryLimited();
  
  /**
   * Force completion of a nonblocking operation locally.
   *
   * Waiting on a nonblocking put or an accumulate operation assures that data
   * was injected into the network and the user buffer can be now be reused.
   * Completing a get operation assures data has arrived into the user memory
   * and is ready for use. Wait operation ensures only local completion. Unlike
   * their blocking counterparts, the nonblocking operations are not ordered
   * with respect to the destination. Performance being one reason, the other
   * reason is that by ensuring ordering we incur additional and possibly
   * unnecessary overhead on applications that do not require their operations
   * to be ordered. For cases where ordering is necessary, it can be done by
   * calling a fence operation.  The fence operation is provided to the user to
   * confirm remote completion if needed.
   *
   * This is a local operation.
   *
   * @param[in] nbhandle nonblocking handle
   */
  void nbWait(GANbhdl *nbhandle);
  
  /**
   * Returns the GA process id (0, ..., ga_Nnodes()-1) of the requesting 
   * compute process.
   *
   * This operation is local. 
   *
   * @return the GA process ID of the requesting process
   */
  int nodeid();
  
  /**
   * Returns the number of the GA compute (user) processes. 
   *
   * This operation is local. 
   *
   * @return the number of GA processes
   */
  int nodes();
  
  /** 
   * Print statistical information on GA use.
   *
   * This non-collective (MIMD) operation prints information about:
   *   - number of calls to
   *     - create
   *     - duplicate
   *     - destroy
   *     - get
   *     - put
   *     - scatter
   *     - gather
   *     - read_and_inc operations
   *   - total amount of data moved in the primitive operations
   *   - amount of data moved in the primitive operations to logicaly remote
   *     locations
   *   - maximum memory consumption in global arrays
   *   - number of requests serviced in the interrupt-driven implementations
   *     by the calling process.
   *
   * This operation is local. 
   */
  void printStats();

  /**
   * Add a user defined data type to GA
   *
   * @param[in] size - size (in bytes) of user defined data type
   *
   * @return  handle for new data type
   */
  int registerType(size_t size);
  
  /**
   * This function sets an internal flag in the GA library to either true or
   * false.
   *
   * The value of this flag can be recovered at any time using the
   * getDebug function. The flag is set to false when the the GA library
   * is initialized. This can be useful in a number of debugging situations,
   * especially when examining the behavior of routines that are called in
   * multiple locations in a code. 
   *
   * This is a local operation.
   *
   * @param[in] dbg value to set internal flag
   */
  void setDebug(int dbg);
  
  /**
   * Sets the amount of memory to be used (in bytes) per process.
   * 
   * This is a local operation. 
   *
   * @param[in] limit the amount of memory in bytes per process
   */
  void setMemoryLimit(size_t limit);
  
  /** 
   * Prints info about allocated arrays. 
   *
   * @param[in] verbose if true print distribution info
   */
  void summarize(int verbose);
  
  /**
   * Synchronize processes (a barrier) and ensure that all GA operations 
   * completed. 
   *
   * This is a collective operation. 
   */
  void sync();
  
  /**
   * Unlocks a mutex object identified by the mutex number.
   *
   * It is a fatal error for a process to attempt to unlock a mutex which has
   * not been locked by this process. 
   *
   * @param[in] mutex object id
   */
  void unlock(int mutex);
  
  /**
   * Returns whether memory comes from internal or external allocator.
   *
   * This operation is local. 
   *
   * @return "1" if memory comes from MA;
   *         "0" if memory comes from another source e.g. System V shared memory
   */
  int usesMA();
  
  /**
   * Returns whether GA is using Fortran indexing.
   *
   * @return "1" if uses fortran API, else returns "0"
   */
  int usesFAPI();
  
  /**
   * This function return a wall (or elapsed) time on the calling processor.
   *
   * Returns time in seconds representing elapsed wall-clock time
   * since an arbitrary time in the past. Example:
   *
   * @code
   *     double starttime, endtime;
   *     starttime = GA::SERVICES.wtime();
   *     // {{.... code snippet to be timed ....}}
   *     endtime   = GA::SERVICES.wtime();
   *     printf("Time taken = %lf seconds\n", endtime-starttime);
   * @endcode
   *
   * This is a local operation.
   *
   * @note This function is only available in release 4.1 or greater.
   */
  double wtime();
};

extern GAServices SERVICES;

}

#endif /* _GA_SERVICES_H */