
/*** DO NOT EDIT THIS FILE DIRECTLY (use 'headers' to generate) ***/


#ifndef hypre_UTILITIES_HEADER
#define hypre_UTILITIES_HEADER

#include "HYPRE_utilities.h"

#ifdef HYPRE_USING_OPENMP
#include <omp.h>
#endif

#ifdef __cplusplus
extern "C" {
#endif

/******************************************************************************
 * Copyright 1998-2019 Lawrence Livermore National Security, LLC and other
 * HYPRE Project Developers. See the top-level COPYRIGHT file for details.
 *
 * SPDX-License-Identifier: (Apache-2.0 OR MIT)
 ******************************************************************************/

/******************************************************************************
 *
 * General structures and values
 *
 *****************************************************************************/

#ifndef hypre_GENERAL_HEADER
#define hypre_GENERAL_HEADER

/* This allows us to consistently avoid 'int' throughout hypre */
typedef int               hypre_int;
typedef long int          hypre_longint;
typedef unsigned int      hypre_uint;
typedef unsigned long int hypre_ulongint;

/* This allows us to consistently avoid 'double' throughout hypre */
typedef double            hypre_double;

/*--------------------------------------------------------------------------
 * Define various functions
 *--------------------------------------------------------------------------*/

#ifndef hypre_max
#define hypre_max(a,b)  (((a)<(b)) ? (b) : (a))
#endif
#ifndef hypre_min
#define hypre_min(a,b)  (((a)<(b)) ? (a) : (b))
#endif

#ifndef hypre_abs
#define hypre_abs(a)  (((a)>0) ? (a) : -(a))
#endif

#ifndef hypre_round
#define hypre_round(x)  ( ((x) < 0.0) ? ((HYPRE_Int)(x - 0.5)) : ((HYPRE_Int)(x + 0.5)) )
#endif

#ifndef hypre_pow2
#define hypre_pow2(i)  ( 1 << (i) )
#endif

#endif /* hypre_GENERAL_HEADER */

/******************************************************************************
 * Copyright 1998-2019 Lawrence Livermore National Security, LLC and other
 * HYPRE Project Developers. See the top-level COPYRIGHT file for details.
 *
 * SPDX-License-Identifier: (Apache-2.0 OR MIT)
 ******************************************************************************/

/******************************************************************************
 *
 *  Fake mpi stubs to generate serial codes without mpi
 *
 *****************************************************************************/

#ifndef hypre_MPISTUBS
#define hypre_MPISTUBS

#ifdef __cplusplus
extern "C" {
#endif

#ifdef HYPRE_SEQUENTIAL

/******************************************************************************
 * MPI stubs to generate serial codes without mpi
 *****************************************************************************/

/*--------------------------------------------------------------------------
 * Change all MPI names to hypre_MPI names to avoid link conflicts.
 *
 * NOTE: MPI_Comm is the only MPI symbol in the HYPRE user interface,
 * and is defined in `HYPRE_utilities.h'.
 *--------------------------------------------------------------------------*/

#define MPI_Comm            hypre_MPI_Comm
#define MPI_Group           hypre_MPI_Group
#define MPI_Request         hypre_MPI_Request
#define MPI_Datatype        hypre_MPI_Datatype
#define MPI_Status          hypre_MPI_Status
#define MPI_Op              hypre_MPI_Op
#define MPI_Aint            hypre_MPI_Aint
#define MPI_Info            hypre_MPI_Info

#define MPI_COMM_WORLD       hypre_MPI_COMM_WORLD
#define MPI_COMM_NULL        hypre_MPI_COMM_NULL
#define MPI_COMM_SELF        hypre_MPI_COMM_SELF
#define MPI_COMM_TYPE_SHARED hypre_MPI_COMM_TYPE_SHARED

#define MPI_BOTTOM  	    hypre_MPI_BOTTOM

#define MPI_FLOAT           hypre_MPI_FLOAT
#define MPI_DOUBLE          hypre_MPI_DOUBLE
#define MPI_LONG_DOUBLE     hypre_MPI_LONG_DOUBLE
#define MPI_INT             hypre_MPI_INT
#define MPI_LONG_LONG_INT   hypre_MPI_INT
#define MPI_CHAR            hypre_MPI_CHAR
#define MPI_LONG            hypre_MPI_LONG
#define MPI_BYTE            hypre_MPI_BYTE
#define MPI_C_DOUBLE_COMPLEX hypre_MPI_COMPLEX

#define MPI_SUM             hypre_MPI_SUM
#define MPI_MIN             hypre_MPI_MIN
#define MPI_MAX             hypre_MPI_MAX
#define MPI_LOR             hypre_MPI_LOR
#define MPI_LAND            hypre_MPI_LAND
#define MPI_SUCCESS         hypre_MPI_SUCCESS
#define MPI_STATUSES_IGNORE hypre_MPI_STATUSES_IGNORE

#define MPI_UNDEFINED       hypre_MPI_UNDEFINED
#define MPI_REQUEST_NULL    hypre_MPI_REQUEST_NULL
#define MPI_INFO_NULL       hypre_MPI_INFO_NULL
#define MPI_ANY_SOURCE      hypre_MPI_ANY_SOURCE
#define MPI_ANY_TAG         hypre_MPI_ANY_TAG
#define MPI_SOURCE          hypre_MPI_SOURCE
#define MPI_TAG             hypre_MPI_TAG

#define MPI_Init            hypre_MPI_Init
#define MPI_Finalize        hypre_MPI_Finalize
#define MPI_Abort           hypre_MPI_Abort
#define MPI_Wtime           hypre_MPI_Wtime
#define MPI_Wtick           hypre_MPI_Wtick
#define MPI_Barrier         hypre_MPI_Barrier
#define MPI_Comm_create     hypre_MPI_Comm_create
#define MPI_Comm_dup        hypre_MPI_Comm_dup
#define MPI_Comm_f2c        hypre_MPI_Comm_f2c
#define MPI_Comm_group      hypre_MPI_Comm_group
#define MPI_Comm_size       hypre_MPI_Comm_size
#define MPI_Comm_rank       hypre_MPI_Comm_rank
#define MPI_Comm_free       hypre_MPI_Comm_free
#define MPI_Comm_split      hypre_MPI_Comm_split
#define MPI_Comm_split_type hypre_MPI_Comm_split_type
#define MPI_Group_incl      hypre_MPI_Group_incl
#define MPI_Group_free      hypre_MPI_Group_free
#define MPI_Address         hypre_MPI_Address
#define MPI_Get_count       hypre_MPI_Get_count
#define MPI_Alltoall        hypre_MPI_Alltoall
#define MPI_Allgather       hypre_MPI_Allgather
#define MPI_Allgatherv      hypre_MPI_Allgatherv
#define MPI_Gather          hypre_MPI_Gather
#define MPI_Gatherv         hypre_MPI_Gatherv
#define MPI_Scatter         hypre_MPI_Scatter
#define MPI_Scatterv        hypre_MPI_Scatterv
#define MPI_Bcast           hypre_MPI_Bcast
#define MPI_Send            hypre_MPI_Send
#define MPI_Recv            hypre_MPI_Recv
#define MPI_Isend           hypre_MPI_Isend
#define MPI_Irecv           hypre_MPI_Irecv
#define MPI_Send_init       hypre_MPI_Send_init
#define MPI_Recv_init       hypre_MPI_Recv_init
#define MPI_Irsend          hypre_MPI_Irsend
#define MPI_Startall        hypre_MPI_Startall
#define MPI_Probe           hypre_MPI_Probe
#define MPI_Iprobe          hypre_MPI_Iprobe
#define MPI_Test            hypre_MPI_Test
#define MPI_Testall         hypre_MPI_Testall
#define MPI_Wait            hypre_MPI_Wait
#define MPI_Waitall         hypre_MPI_Waitall
#define MPI_Waitany         hypre_MPI_Waitany
#define MPI_Allreduce       hypre_MPI_Allreduce
#define MPI_Reduce          hypre_MPI_Reduce
#define MPI_Scan            hypre_MPI_Scan
#define MPI_Request_free    hypre_MPI_Request_free
#define MPI_Type_contiguous hypre_MPI_Type_contiguous
#define MPI_Type_vector     hypre_MPI_Type_vector
#define MPI_Type_hvector    hypre_MPI_Type_hvector
#define MPI_Type_struct     hypre_MPI_Type_struct
#define MPI_Type_commit     hypre_MPI_Type_commit
#define MPI_Type_free       hypre_MPI_Type_free
#define MPI_Op_free         hypre_MPI_Op_free
#define MPI_Op_create       hypre_MPI_Op_create
#define MPI_User_function   hypre_MPI_User_function
#define MPI_Info_create     hypre_MPI_Info_create

/*--------------------------------------------------------------------------
 * Types, etc.
 *--------------------------------------------------------------------------*/

/* These types have associated creation and destruction routines */
typedef HYPRE_Int hypre_MPI_Comm;
typedef HYPRE_Int hypre_MPI_Group;
typedef HYPRE_Int hypre_MPI_Request;
typedef HYPRE_Int hypre_MPI_Datatype;
typedef void (hypre_MPI_User_function) ();

typedef struct
{
   HYPRE_Int hypre_MPI_SOURCE;
   HYPRE_Int hypre_MPI_TAG;
} hypre_MPI_Status;

typedef HYPRE_Int  hypre_MPI_Op;
typedef HYPRE_Int  hypre_MPI_Aint;
typedef HYPRE_Int  hypre_MPI_Info;

#define  hypre_MPI_COMM_SELF   1
#define  hypre_MPI_COMM_WORLD  0
#define  hypre_MPI_COMM_NULL  -1

#define  hypre_MPI_COMM_TYPE_SHARED 0

#define  hypre_MPI_BOTTOM  0x0

#define  hypre_MPI_FLOAT 0
#define  hypre_MPI_DOUBLE 1
#define  hypre_MPI_LONG_DOUBLE 2
#define  hypre_MPI_INT 3
#define  hypre_MPI_CHAR 4
#define  hypre_MPI_LONG 5
#define  hypre_MPI_BYTE 6
#define  hypre_MPI_REAL 7
#define  hypre_MPI_COMPLEX 8

#define  hypre_MPI_SUM 0
#define  hypre_MPI_MIN 1
#define  hypre_MPI_MAX 2
#define  hypre_MPI_LOR 3
#define  hypre_MPI_LAND 4
#define  hypre_MPI_SUCCESS 0
#define  hypre_MPI_STATUSES_IGNORE 0

#define  hypre_MPI_UNDEFINED -9999
#define  hypre_MPI_REQUEST_NULL  0
#define  hypre_MPI_INFO_NULL     0
#define  hypre_MPI_ANY_SOURCE    1
#define  hypre_MPI_ANY_TAG       1

#else

/******************************************************************************
 * MPI stubs to do casting of HYPRE_Int and hypre_int correctly
 *****************************************************************************/

typedef MPI_Comm     hypre_MPI_Comm;
typedef MPI_Group    hypre_MPI_Group;
typedef MPI_Request  hypre_MPI_Request;
typedef MPI_Datatype hypre_MPI_Datatype;
typedef MPI_Status   hypre_MPI_Status;
typedef MPI_Op       hypre_MPI_Op;
typedef MPI_Aint     hypre_MPI_Aint;
typedef MPI_Info     hypre_MPI_Info;
typedef MPI_User_function    hypre_MPI_User_function;

#define  hypre_MPI_COMM_WORLD         MPI_COMM_WORLD
#define  hypre_MPI_COMM_NULL          MPI_COMM_NULL
#define  hypre_MPI_BOTTOM             MPI_BOTTOM
#define  hypre_MPI_COMM_SELF          MPI_COMM_SELF
#define  hypre_MPI_COMM_TYPE_SHARED   MPI_COMM_TYPE_SHARED

#define  hypre_MPI_FLOAT   MPI_FLOAT
#define  hypre_MPI_DOUBLE  MPI_DOUBLE
#define  hypre_MPI_LONG_DOUBLE  MPI_LONG_DOUBLE
/* HYPRE_MPI_INT is defined in HYPRE_utilities.h */
#define  hypre_MPI_INT     HYPRE_MPI_INT
#define  hypre_MPI_CHAR    MPI_CHAR
#define  hypre_MPI_LONG    MPI_LONG
#define  hypre_MPI_BYTE    MPI_BYTE
/* HYPRE_MPI_REAL is defined in HYPRE_utilities.h */
#define  hypre_MPI_REAL    HYPRE_MPI_REAL
/* HYPRE_MPI_COMPLEX is defined in HYPRE_utilities.h */
#define  hypre_MPI_COMPLEX HYPRE_MPI_COMPLEX

#define  hypre_MPI_SUM MPI_SUM
#define  hypre_MPI_MIN MPI_MIN
#define  hypre_MPI_MAX MPI_MAX
#define  hypre_MPI_LOR MPI_LOR
#define  hypre_MPI_SUCCESS MPI_SUCCESS
#define  hypre_MPI_STATUSES_IGNORE MPI_STATUSES_IGNORE

#define  hypre_MPI_UNDEFINED       MPI_UNDEFINED
#define  hypre_MPI_REQUEST_NULL    MPI_REQUEST_NULL
#define  hypre_MPI_INFO_NULL       MPI_INFO_NULL
#define  hypre_MPI_ANY_SOURCE      MPI_ANY_SOURCE
#define  hypre_MPI_ANY_TAG         MPI_ANY_TAG
#define  hypre_MPI_SOURCE          MPI_SOURCE
#define  hypre_MPI_TAG             MPI_TAG
#define  hypre_MPI_LAND            MPI_LAND

#endif

/******************************************************************************
 * Everything below this applies to both ifdef cases above
 *****************************************************************************/

/*--------------------------------------------------------------------------
 * Prototypes
 *--------------------------------------------------------------------------*/

/* mpistubs.c */
HYPRE_Int hypre_MPI_Init( hypre_int *argc , char ***argv );
HYPRE_Int hypre_MPI_Finalize( void );
HYPRE_Int hypre_MPI_Abort( hypre_MPI_Comm comm , HYPRE_Int errorcode );
HYPRE_Real hypre_MPI_Wtime( void );
HYPRE_Real hypre_MPI_Wtick( void );
HYPRE_Int hypre_MPI_Barrier( hypre_MPI_Comm comm );
HYPRE_Int hypre_MPI_Comm_create( hypre_MPI_Comm comm , hypre_MPI_Group group , hypre_MPI_Comm *newcomm );
HYPRE_Int hypre_MPI_Comm_dup( hypre_MPI_Comm comm , hypre_MPI_Comm *newcomm );
hypre_MPI_Comm hypre_MPI_Comm_f2c( hypre_int comm );
HYPRE_Int hypre_MPI_Comm_size( hypre_MPI_Comm comm , HYPRE_Int *size );
HYPRE_Int hypre_MPI_Comm_rank( hypre_MPI_Comm comm , HYPRE_Int *rank );
HYPRE_Int hypre_MPI_Comm_free( hypre_MPI_Comm *comm );
HYPRE_Int hypre_MPI_Comm_group( hypre_MPI_Comm comm , hypre_MPI_Group *group );
HYPRE_Int hypre_MPI_Comm_split( hypre_MPI_Comm comm, HYPRE_Int n, HYPRE_Int m, hypre_MPI_Comm * comms );
HYPRE_Int hypre_MPI_Group_incl( hypre_MPI_Group group , HYPRE_Int n , HYPRE_Int *ranks , hypre_MPI_Group *newgroup );
HYPRE_Int hypre_MPI_Group_free( hypre_MPI_Group *group );
HYPRE_Int hypre_MPI_Address( void *location , hypre_MPI_Aint *address );
HYPRE_Int hypre_MPI_Get_count( hypre_MPI_Status *status , hypre_MPI_Datatype datatype , HYPRE_Int *count );
HYPRE_Int hypre_MPI_Alltoall( void *sendbuf , HYPRE_Int sendcount , hypre_MPI_Datatype sendtype , void *recvbuf , HYPRE_Int recvcount , hypre_MPI_Datatype recvtype , hypre_MPI_Comm comm );
HYPRE_Int hypre_MPI_Allgather( void *sendbuf , HYPRE_Int sendcount , hypre_MPI_Datatype sendtype , void *recvbuf , HYPRE_Int recvcount , hypre_MPI_Datatype recvtype , hypre_MPI_Comm comm );
HYPRE_Int hypre_MPI_Allgatherv( void *sendbuf , HYPRE_Int sendcount , hypre_MPI_Datatype sendtype , void *recvbuf , HYPRE_Int *recvcounts , HYPRE_Int *displs , hypre_MPI_Datatype recvtype , hypre_MPI_Comm comm );
HYPRE_Int hypre_MPI_Gather( void *sendbuf , HYPRE_Int sendcount , hypre_MPI_Datatype sendtype , void *recvbuf , HYPRE_Int recvcount , hypre_MPI_Datatype recvtype , HYPRE_Int root , hypre_MPI_Comm comm );
HYPRE_Int hypre_MPI_Gatherv( void *sendbuf , HYPRE_Int sendcount , hypre_MPI_Datatype sendtype , void *recvbuf , HYPRE_Int *recvcounts , HYPRE_Int *displs , hypre_MPI_Datatype recvtype , HYPRE_Int root , hypre_MPI_Comm comm );
HYPRE_Int hypre_MPI_Scatter( void *sendbuf , HYPRE_Int sendcount , hypre_MPI_Datatype sendtype , void *recvbuf , HYPRE_Int recvcount , hypre_MPI_Datatype recvtype , HYPRE_Int root , hypre_MPI_Comm comm );
HYPRE_Int hypre_MPI_Scatterv( void *sendbuf , HYPRE_Int *sendcounts , HYPRE_Int *displs, hypre_MPI_Datatype sendtype , void *recvbuf , HYPRE_Int recvcount , hypre_MPI_Datatype recvtype , HYPRE_Int root , hypre_MPI_Comm comm );
HYPRE_Int hypre_MPI_Bcast( void *buffer , HYPRE_Int count , hypre_MPI_Datatype datatype , HYPRE_Int root , hypre_MPI_Comm comm );
HYPRE_Int hypre_MPI_Send( void *buf , HYPRE_Int count , hypre_MPI_Datatype datatype , HYPRE_Int dest , HYPRE_Int tag , hypre_MPI_Comm comm );
HYPRE_Int hypre_MPI_Recv( void *buf , HYPRE_Int count , hypre_MPI_Datatype datatype , HYPRE_Int source , HYPRE_Int tag , hypre_MPI_Comm comm , hypre_MPI_Status *status );
HYPRE_Int hypre_MPI_Isend( void *buf , HYPRE_Int count , hypre_MPI_Datatype datatype , HYPRE_Int dest , HYPRE_Int tag , hypre_MPI_Comm comm , hypre_MPI_Request *request );
HYPRE_Int hypre_MPI_Irecv( void *buf , HYPRE_Int count , hypre_MPI_Datatype datatype , HYPRE_Int source , HYPRE_Int tag , hypre_MPI_Comm comm , hypre_MPI_Request *request );
HYPRE_Int hypre_MPI_Send_init( void *buf , HYPRE_Int count , hypre_MPI_Datatype datatype , HYPRE_Int dest , HYPRE_Int tag , hypre_MPI_Comm comm , hypre_MPI_Request *request );
HYPRE_Int hypre_MPI_Recv_init( void *buf , HYPRE_Int count , hypre_MPI_Datatype datatype , HYPRE_Int dest , HYPRE_Int tag , hypre_MPI_Comm comm , hypre_MPI_Request *request );
HYPRE_Int hypre_MPI_Irsend( void *buf , HYPRE_Int count , hypre_MPI_Datatype datatype , HYPRE_Int dest , HYPRE_Int tag , hypre_MPI_Comm comm , hypre_MPI_Request *request );
HYPRE_Int hypre_MPI_Startall( HYPRE_Int count , hypre_MPI_Request *array_of_requests );
HYPRE_Int hypre_MPI_Probe( HYPRE_Int source , HYPRE_Int tag , hypre_MPI_Comm comm , hypre_MPI_Status *status );
HYPRE_Int hypre_MPI_Iprobe( HYPRE_Int source , HYPRE_Int tag , hypre_MPI_Comm comm , HYPRE_Int *flag , hypre_MPI_Status *status );
HYPRE_Int hypre_MPI_Test( hypre_MPI_Request *request , HYPRE_Int *flag , hypre_MPI_Status *status );
HYPRE_Int hypre_MPI_Testall( HYPRE_Int count , hypre_MPI_Request *array_of_requests , HYPRE_Int *flag , hypre_MPI_Status *array_of_statuses );
HYPRE_Int hypre_MPI_Wait( hypre_MPI_Request *request , hypre_MPI_Status *status );
HYPRE_Int hypre_MPI_Waitall( HYPRE_Int count , hypre_MPI_Request *array_of_requests , hypre_MPI_Status *array_of_statuses );
HYPRE_Int hypre_MPI_Waitany( HYPRE_Int count , hypre_MPI_Request *array_of_requests , HYPRE_Int *index , hypre_MPI_Status *status );
HYPRE_Int hypre_MPI_Allreduce( void *sendbuf , void *recvbuf , HYPRE_Int count , hypre_MPI_Datatype datatype , hypre_MPI_Op op , hypre_MPI_Comm comm );
HYPRE_Int hypre_MPI_Reduce( void *sendbuf , void *recvbuf , HYPRE_Int count , hypre_MPI_Datatype datatype , hypre_MPI_Op op , HYPRE_Int root , hypre_MPI_Comm comm );
HYPRE_Int hypre_MPI_Scan( void *sendbuf , void *recvbuf , HYPRE_Int count , hypre_MPI_Datatype datatype , hypre_MPI_Op op , hypre_MPI_Comm comm );
HYPRE_Int hypre_MPI_Request_free( hypre_MPI_Request *request );
HYPRE_Int hypre_MPI_Type_contiguous( HYPRE_Int count , hypre_MPI_Datatype oldtype , hypre_MPI_Datatype *newtype );
HYPRE_Int hypre_MPI_Type_vector( HYPRE_Int count , HYPRE_Int blocklength , HYPRE_Int stride , hypre_MPI_Datatype oldtype , hypre_MPI_Datatype *newtype );
HYPRE_Int hypre_MPI_Type_hvector( HYPRE_Int count , HYPRE_Int blocklength , hypre_MPI_Aint stride , hypre_MPI_Datatype oldtype , hypre_MPI_Datatype *newtype );
HYPRE_Int hypre_MPI_Type_struct( HYPRE_Int count , HYPRE_Int *array_of_blocklengths , hypre_MPI_Aint *array_of_displacements , hypre_MPI_Datatype *array_of_types , hypre_MPI_Datatype *newtype );
HYPRE_Int hypre_MPI_Type_commit( hypre_MPI_Datatype *datatype );
HYPRE_Int hypre_MPI_Type_free( hypre_MPI_Datatype *datatype );
HYPRE_Int hypre_MPI_Op_free( hypre_MPI_Op *op );
HYPRE_Int hypre_MPI_Op_create( hypre_MPI_User_function *function , hypre_int commute , hypre_MPI_Op *op );
#if defined(HYPRE_USING_CUDA) || defined(HYPRE_USING_DEVICE_OPENMP)
HYPRE_Int hypre_MPI_Comm_split_type(hypre_MPI_Comm comm, HYPRE_Int split_type, HYPRE_Int key, hypre_MPI_Info info, hypre_MPI_Comm *newcomm);
HYPRE_Int hypre_MPI_Info_create(hypre_MPI_Info *info);
HYPRE_Int hypre_MPI_Info_free( hypre_MPI_Info *info );
#endif

#ifdef __cplusplus
}
#endif

#endif

/******************************************************************************
 * Copyright 1998-2019 Lawrence Livermore National Security, LLC and other
 * HYPRE Project Developers. See the top-level COPYRIGHT file for details.
 *
 * SPDX-License-Identifier: (Apache-2.0 OR MIT)
 ******************************************************************************/

#ifndef HYPRE_SMP_HEADER
#define HYPRE_SMP_HEADER
#endif

#define HYPRE_SMP_SCHEDULE schedule(static)

/******************************************************************************
 * Copyright 1998-2019 Lawrence Livermore National Security, LLC and other
 * HYPRE Project Developers. See the top-level COPYRIGHT file for details.
 *
 * SPDX-License-Identifier: (Apache-2.0 OR MIT)
 ******************************************************************************/

/******************************************************************************
 *
 * Header file for memory management utilities
 *
 * The abstract memory model has a Host (think CPU) and a Device (think GPU) and
 * three basic types of memory management utilities:
 *
 *    1. Malloc(..., location)
 *             location=LOCATION_DEVICE - malloc memory on the device
 *             location=LOCATION_HOST   - malloc memory on the host
 *    2. MemCopy(..., method)
 *             method=HOST_TO_DEVICE    - copy from host to device
 *             method=DEVICE_TO_HOST    - copy from device to host
 *             method=DEVICE_TO_DEVICE  - copy from device to device
 *    3. SetExecutionMode
 *             location=LOCATION_DEVICE - execute on the device
 *             location=LOCATION_HOST   - execute on the host
 *
 * Although the abstract model does not explicitly reflect a managed memory
 * model (i.e., unified memory), it can support it.  Here is a summary of how
 * the abstract model would be mapped to specific hardware scenarios:
 *
 *    Not using a device, not using managed memory
 *       Malloc(..., location)
 *             location=LOCATION_DEVICE - host malloc          e.g., malloc
 *             location=LOCATION_HOST   - host malloc          e.g., malloc
 *       MemoryCopy(..., locTo,locFrom)
 *             locTo=LOCATION_HOST,   locFrom=LOCATION_DEVICE  - copy from host to host e.g., memcpy
 *             locTo=LOCATION_DEVICE, locFrom=LOCATION_HOST    - copy from host to host e.g., memcpy
 *             locTo=LOCATION_DEVICE, locFrom=LOCATION_DEVICE  - copy from host to host e.g., memcpy
 *       SetExecutionMode
 *             location=LOCATION_DEVICE - execute on the host
 *             location=LOCATION_HOST   - execute on the host
 *
 *    Using a device, not using managed memory
 *       Malloc(..., location)
 *             location=LOCATION_DEVICE - device malloc        e.g., cudaMalloc
 *             location=LOCATION_HOST   - host malloc          e.g., malloc
 *       MemoryCopy(..., locTo,locFrom)
 *             locTo=LOCATION_HOST,   locFrom=LOCATION_DEVICE  - copy from device to host e.g., cudaMemcpy
 *             locTo=LOCATION_DEVICE, locFrom=LOCATION_HOST    - copy from host to device e.g., cudaMemcpy
 *             locTo=LOCATION_DEVICE, locFrom=LOCATION_DEVICE  - copy from device to device e.g., cudaMemcpy
 *       SetExecutionMode
 *             location=LOCATION_DEVICE - execute on the device
 *             location=LOCATION_HOST   - execute on the host
 *
 *    Using a device, using managed memory
 *       Malloc(..., location)
 *             location=LOCATION_DEVICE - managed malloc        e.g., cudaMallocManaged
 *             location=LOCATION_HOST   - host malloc          e.g., malloc
 *       MemoryCopy(..., locTo,locFrom)
 *             locTo=LOCATION_HOST,   locFrom=LOCATION_DEVICE  - copy from device to host e.g., cudaMallocManaged
 *             locTo=LOCATION_DEVICE, locFrom=LOCATION_HOST    - copy from host to device e.g., cudaMallocManaged
 *             locTo=LOCATION_DEVICE, locFrom=LOCATION_DEVICE  - copy from device to device e.g., cudaMallocManaged
 *       SetExecutionMode
 *             location=LOCATION_DEVICE - execute on the device
 *             location=LOCATION_HOST   - execute on the host
 *
 * Questions:
 *
 *    1. prefetch?
 *
 *****************************************************************************/

#ifndef hypre_MEMORY_HEADER
#define hypre_MEMORY_HEADER

#include <stdio.h>
#include <stdlib.h>

/* stringification:
 * _Pragma(string-literal), so we need to cast argument to a string
 * The three dots as last argument of the macro tells compiler that this is a variadic macro.
 * I.e. this is a macro that receives variable number of arguments.
 */
#define HYPRE_STR(...) #__VA_ARGS__
#define HYPRE_XSTR(...) HYPRE_STR(__VA_ARGS__)

#ifdef __cplusplus
extern "C" {
#endif

#define HYPRE_MEMORY_UNSET         (-1)
#define HYPRE_MEMORY_DEVICE        ( 0)
#define HYPRE_MEMORY_HOST          ( 1)
#define HYPRE_MEMORY_SHARED        ( 2)
#define HYPRE_MEMORY_HOST_PINNED   ( 3)

#define HYPRE_EXEC_UNSET           (-1)
#define HYPRE_EXEC_DEVICE          ( 0)
#define HYPRE_EXEC_HOST            ( 1)

/*==================================================================
 *       default def of memory location selected based memory env
 *   +-------------------------------------------------------------+
 *   |                           |          HYPRE_MEMORY_*         |
 *   |        MEM \ LOC          | HOST | DEVICE | SHARED | PINNED |
 *   |---------------------------+---------------+-----------------|
 *   | HYPRE_USING_HOST_MEMORY   | HOST | HOST   | HOST   | HOST   |
 *   |---------------------------+---------------+-------- --------|
 *   | HYPRE_USING_DEVICE_MEMORY | HOST | DEVICE | DEVICE | PINNED |
 *   |---------------------------+---------------+-----------------|
 *   | HYPRE_USING_UNIFIED_MEMORY| HOST | DEVICE | SHARED | PINNED |
 *   +-------------------------------------------------------------+
 *==================================================================*/

#if defined(HYPRE_USING_HOST_MEMORY)

/* default memory model without device (host only) */
#define HYPRE_MEMORY_HOST_ACT         HYPRE_MEMORY_HOST
#define HYPRE_MEMORY_DEVICE_ACT       HYPRE_MEMORY_HOST
#define HYPRE_MEMORY_SHARED_ACT       HYPRE_MEMORY_HOST
#define HYPRE_MEMORY_HOST_PINNED_ACT  HYPRE_MEMORY_HOST

#elif defined(HYPRE_USING_DEVICE_MEMORY)

/* default memory model with device and without unified memory */
#define HYPRE_MEMORY_HOST_ACT         HYPRE_MEMORY_HOST
#define HYPRE_MEMORY_DEVICE_ACT       HYPRE_MEMORY_DEVICE
#define HYPRE_MEMORY_SHARED_ACT       HYPRE_MEMORY_DEVICE
#define HYPRE_MEMORY_HOST_PINNED_ACT  HYPRE_MEMORY_HOST_PINNED

#elif defined(HYPRE_USING_UNIFIED_MEMORY)

/* default memory model with device and with unified memory */
#define HYPRE_MEMORY_HOST_ACT         HYPRE_MEMORY_HOST
#define HYPRE_MEMORY_DEVICE_ACT       HYPRE_MEMORY_DEVICE
#define HYPRE_MEMORY_SHARED_ACT       HYPRE_MEMORY_SHARED
//#define HYPRE_MEMORY_SHARED_ACT       HYPRE_MEMORY_HOST
#define HYPRE_MEMORY_HOST_PINNED_ACT  HYPRE_MEMORY_HOST_PINNED

#else

/* default */
#define HYPRE_MEMORY_HOST_ACT         HYPRE_MEMORY_HOST
#define HYPRE_MEMORY_DEVICE_ACT       HYPRE_MEMORY_HOST
#define HYPRE_MEMORY_SHARED_ACT       HYPRE_MEMORY_HOST
#define HYPRE_MEMORY_HOST_PINNED_ACT  HYPRE_MEMORY_HOST

#endif

/* the above definitions might be overridden to customize
 * memory locations */

/* #undef  HYPRE_MEMORY_HOST_ACT */
/* #undef  HYPRE_MEMORY_DEVICE_ACT */
/* #undef  HYPRE_MEMORY_SHARED_ACT */
/* #undef  HYPRE_MEMORY_PINNED_ACT */
/* #define HYPRE_MEMORY_HOST_ACT    HYPRE_MEMORY_? */
/* #define HYPRE_MEMORY_DEVICE_ACT  HYPRE_MEMORY_? */
/* #define HYPRE_MEMORY_SHARED_ACT  HYPRE_MEMORY_? */
/* #define HYPRE_MEMORY_PINNED_ACT  HYPRE_MEMORY_? */

/*-------------------------------------------------------
 * hypre_GetActualMemLocation
 *   return actual location based on the selected memory model
 *-------------------------------------------------------*/
static inline HYPRE_Int
hypre_GetActualMemLocation(HYPRE_Int location)
{
   if (location == HYPRE_MEMORY_HOST)
   {
      return HYPRE_MEMORY_HOST_ACT;
   }

   if (location == HYPRE_MEMORY_DEVICE)
   {
      return HYPRE_MEMORY_DEVICE_ACT;
   }

   if (location == HYPRE_MEMORY_SHARED)
   {
      return HYPRE_MEMORY_SHARED_ACT;
   }

   if (location == HYPRE_MEMORY_HOST_PINNED)
   {
      return HYPRE_MEMORY_HOST_PINNED_ACT;
   }

   return HYPRE_MEMORY_UNSET;
}

#define HYPRE_MEM_PAD_LEN 1

#if 0
/* These Allocs are with printfs, for debug */
#define hypre_TAlloc(type, count, location) \
(\
{\
 if (location == HYPRE_MEMORY_DEVICE) printf("[%s:%d] TALLOC %.3f MB\n", __FILE__,__LINE__, (size_t)(sizeof(type) * (count))/1024.0/1024.0); \
 (type *) hypre_MAlloc((size_t)(sizeof(type) * (count)), location); \
}\
)

#define hypre_CTAlloc(type, count, location) \
(\
{\
 if (location == HYPRE_MEMORY_DEVICE) printf("[%s:%d] CTALLOC %.3f MB\n", __FILE__,__LINE__, (size_t)(sizeof(type) * (count))/1024.0/1024.0); \
 (type *) hypre_CAlloc((size_t)(count), (size_t)sizeof(type), location); \
}\
)

#define hypre_TReAlloc(ptr, type, count, location) \
(\
{\
 if (location == HYPRE_MEMORY_DEVICE) printf("[%s:%d] TReALLOC %p, %.3f MB\n", __FILE__,__LINE__, ptr, (size_t)(sizeof(type) * (count))/1024.0/1024.0); \
 (type *)hypre_ReAlloc((char *)ptr, (size_t)(sizeof(type) * (count)), location); \
}\
)

#define hypre_TMemcpy(dst, src, type, count, locdst, locsrc) \
( \
{ \
  printf("[%s:%d] TMemcpy %d to %d %.3f MB\n", __FILE__,__LINE__, locsrc, locdst, (size_t)(sizeof(type) * (count))/1024.0/1024.0); \
  hypre_Memcpy((void *)(dst), (void *)(src), (size_t)(sizeof(type) * (count)), locdst, locsrc); \
} \
)

#else

#define hypre_TAlloc(type, count, location) \
( (type *) hypre_MAlloc((size_t)(sizeof(type) * (count)), location) )

#define hypre_CTAlloc(type, count, location) \
( (type *) hypre_CAlloc((size_t)(count), (size_t)sizeof(type), location) )

#define hypre_TReAlloc(ptr, type, count, location) \
( (type *) hypre_ReAlloc((char *)ptr, (size_t)(sizeof(type) * (count)), location) )

#define hypre_TMemcpy(dst, src, type, count, locdst, locsrc) \
(hypre_Memcpy((void *)(dst), (void *)(src), (size_t)(sizeof(type) * (count)), locdst, locsrc))

#endif

#define hypre_TFree(ptr, location) \
( hypre_Free((void *)ptr, location), ptr = NULL )

/*--------------------------------------------------------------------------
 * Prototypes
 *--------------------------------------------------------------------------*/

/* hypre_memory.c */
void * hypre_MAlloc(size_t size, HYPRE_Int location);
void * hypre_CAlloc( size_t count, size_t elt_size, HYPRE_Int location);
void * hypre_ReAlloc(void *ptr, size_t size, HYPRE_Int location);
void   hypre_Memcpy(void *dst, void *src, size_t size, HYPRE_Int loc_dst, HYPRE_Int loc_src);
void * hypre_Memset(void *ptr, HYPRE_Int value, size_t num, HYPRE_Int location);
void   hypre_Free(void *ptr, HYPRE_Int location);
HYPRE_Int hypre_GetMemoryLocation(const void *ptr, HYPRE_Int *memory_location);

/* memory_dmalloc.c */
HYPRE_Int hypre_InitMemoryDebugDML( HYPRE_Int id );
HYPRE_Int hypre_FinalizeMemoryDebugDML( void );
char *hypre_MAllocDML( HYPRE_Int size , char *file , HYPRE_Int line );
char *hypre_CAllocDML( HYPRE_Int count , HYPRE_Int elt_size , char *file , HYPRE_Int line );
char *hypre_ReAllocDML( char *ptr , HYPRE_Int size , char *file , HYPRE_Int line );
void hypre_FreeDML( char *ptr , char *file , HYPRE_Int line );

#ifdef __cplusplus
}
#endif

#endif

/******************************************************************************
 * Copyright 1998-2019 Lawrence Livermore National Security, LLC and other
 * HYPRE Project Developers. See the top-level COPYRIGHT file for details.
 *
 * SPDX-License-Identifier: (Apache-2.0 OR MIT)
 ******************************************************************************/

#ifndef HYPRE_OMP_DEVICE_H
#define HYPRE_OMP_DEVICE_H

#if defined(HYPRE_USING_DEVICE_OPENMP)

#include "omp.h"

/* OpenMP 4.5 device memory management */
extern HYPRE_Int hypre__global_offload;
extern HYPRE_Int hypre__offload_device_num;
extern HYPRE_Int hypre__offload_host_num;

/* stats */
extern size_t hypre__target_allc_count;
extern size_t hypre__target_free_count;
extern size_t hypre__target_allc_bytes;
extern size_t hypre__target_free_bytes;
extern size_t hypre__target_htod_count;
extern size_t hypre__target_dtoh_count;
extern size_t hypre__target_htod_bytes;
extern size_t hypre__target_dtoh_bytes;

/* CHECK MODE: check if offloading has effect (turned on when configured with --enable-debug)
 * if we ``enter'' an address, it should not exist in device [o.w NO EFFECT]
 * if we ``exit'' or ''update'' an address, it should exist in device [o.w ERROR]
 * hypre__offload_flag: 0 == OK; 1 == WRONG
 */
#ifdef HYPRE_DEVICE_OPENMP_CHECK
#define HYPRE_OFFLOAD_FLAG(devnum, hptr, type) HYPRE_Int hypre__offload_flag = (type[1] == 'n') == omp_target_is_present(hptr, devnum);
#else
#define HYPRE_OFFLOAD_FLAG(...) HYPRE_Int hypre__offload_flag = 0; /* non-debug mode, always OK */
#endif

/* OMP 4.5 offloading macro */
#define hypre_omp45_offload(devnum, hptr, datatype, offset, count, type1, type2) \
{\
   /* devnum: device number \
    * hptr: host poiter \
    * datatype \
    * type1: ``e(n)ter'', ''e(x)it'', or ``u(p)date'' \
    * type2: ``(a)lloc'', ``(t)o'', ``(d)elete'', ''(f)rom'' \
    */ \
   datatype *hypre__offload_hptr = (datatype *) hptr; \
   /* if hypre__global_offload ==    0, or
    *    hptr (host pointer)   == NULL,
    *    this offload will be IGNORED */ \
   if (hypre__global_offload && hypre__offload_hptr != NULL) { \
      /* offloading offset and size (in datatype) */ \
      size_t hypre__offload_offset = offset, hypre__offload_size = count; \
      /* in the CHECK mode, we test if this offload has effect */ \
      HYPRE_OFFLOAD_FLAG(devnum, hypre__offload_hptr, type1) \
      if (hypre__offload_flag) { \
         printf("[!NO Effect! %s %d] device %d target: %6s %6s, data %p, [%ld:%ld]\n", __FILE__, __LINE__, devnum, type1, type2, (void *)hypre__offload_hptr, hypre__offload_offset, hypre__offload_size); exit(0); \
      } else { \
         size_t offload_bytes = count * sizeof(datatype); \
         /* printf("[            %s %d] device %d target: %6s %6s, data %p, [%d:%d]\n", __FILE__, __LINE__, devnum, type1, type2, (void *)hypre__offload_hptr, hypre__offload_offset, hypre__offload_size); */ \
         if (type1[1] == 'n' && type2[0] == 't') { \
            /* enter to */\
            hypre__target_allc_count ++; \
            hypre__target_allc_bytes += offload_bytes; \
            hypre__target_htod_count ++; \
            hypre__target_htod_bytes += offload_bytes; \
            _Pragma (HYPRE_XSTR(omp target enter data map(to:hypre__offload_hptr[hypre__offload_offset:hypre__offload_size]))) \
         } else if (type1[1] == 'n' && type2[0] == 'a') { \
            /* enter alloc */ \
            hypre__target_allc_count ++; \
            hypre__target_allc_bytes += offload_bytes; \
            _Pragma (HYPRE_XSTR(omp target enter data map(alloc:hypre__offload_hptr[hypre__offload_offset:hypre__offload_size]))) \
         } else if (type1[1] == 'x' && type2[0] == 'd') { \
            /* exit delete */\
            hypre__target_free_count ++; \
            hypre__target_free_bytes += offload_bytes; \
            _Pragma (HYPRE_XSTR(omp target exit data map(delete:hypre__offload_hptr[hypre__offload_offset:hypre__offload_size]))) \
         } else if (type1[1] == 'x' && type2[0] == 'f') {\
            /* exit from */ \
            hypre__target_free_count ++; \
            hypre__target_free_bytes += offload_bytes; \
            hypre__target_dtoh_count ++; \
            hypre__target_dtoh_bytes += offload_bytes; \
            _Pragma (HYPRE_XSTR(omp target exit data map(from:hypre__offload_hptr[hypre__offload_offset:hypre__offload_size]))) \
         } else if (type1[1] == 'p' && type2[0] == 't') { \
            /* update to */ \
            hypre__target_htod_count ++; \
            hypre__target_htod_bytes += offload_bytes; \
            _Pragma (HYPRE_XSTR(omp target update to(hypre__offload_hptr[hypre__offload_offset:hypre__offload_size]))) \
         } else if (type1[1] == 'p' && type2[0] == 'f') {\
            /* update from */ \
            hypre__target_dtoh_count ++; \
            hypre__target_dtoh_bytes += offload_bytes; \
            _Pragma (HYPRE_XSTR(omp target update from(hypre__offload_hptr[hypre__offload_offset:hypre__offload_size]))) \
         } else {\
            printf("error: unrecognized offloading type combination!\n"); exit(-1); \
         } \
      } \
   } \
}

HYPRE_Int HYPRE_OMPOffload(HYPRE_Int device, void *ptr, size_t num, const char *type1, const char *type2);
HYPRE_Int HYPRE_OMPPtrIsMapped(void *p, HYPRE_Int device_num);
HYPRE_Int HYPRE_OMPOffloadOn();
HYPRE_Int HYPRE_OMPOffloadOff();
HYPRE_Int HYPRE_OMPOffloadStatPrint();

#define HYPRE_MIN_GPU_SIZE (131072)

#define hypre_SetDeviceOn() HYPRE_OMPOffloadOn()
#define hypre_SetDeviceOff() HYPRE_OMPOffloadOff()

#endif /* HYPRE_USING_DEVICE_OPENMP */
#endif /* HYPRE_OMP_DEVICE_H */

/******************************************************************************
 * Copyright 1998-2019 Lawrence Livermore National Security, LLC and other
 * HYPRE Project Developers. See the top-level COPYRIGHT file for details.
 *
 * SPDX-License-Identifier: (Apache-2.0 OR MIT)
 ******************************************************************************/

#ifndef hypre_THREADING_HEADER
#define hypre_THREADING_HEADER

#ifdef HYPRE_USING_OPENMP

HYPRE_Int hypre_NumThreads( void );
HYPRE_Int hypre_NumActiveThreads( void );
HYPRE_Int hypre_GetThreadNum( void );
void      hypre_SetNumThreads(HYPRE_Int nt);

#else

#define hypre_NumThreads() 1
#define hypre_NumActiveThreads() 1
#define hypre_GetThreadNum() 0
#define hypre_SetNumThreads(x)

#endif

void hypre_GetSimpleThreadPartition( HYPRE_Int *begin, HYPRE_Int *end, HYPRE_Int n );

#endif

/******************************************************************************
 * Copyright 1998-2019 Lawrence Livermore National Security, LLC and other
 * HYPRE Project Developers. See the top-level COPYRIGHT file for details.
 *
 * SPDX-License-Identifier: (Apache-2.0 OR MIT)
 ******************************************************************************/

/******************************************************************************
 *
 * Header file for doing timing
 *
 *****************************************************************************/

#ifndef HYPRE_TIMING_HEADER
#define HYPRE_TIMING_HEADER

#include <stdlib.h>
#include <stdio.h>
#include <string.h>

#ifdef __cplusplus
extern "C" {
#endif

/*--------------------------------------------------------------------------
 * Prototypes for low-level timing routines
 *--------------------------------------------------------------------------*/

/* timer.c */
HYPRE_Real time_getWallclockSeconds( void );
HYPRE_Real time_getCPUSeconds( void );
HYPRE_Real time_get_wallclock_seconds_( void );
HYPRE_Real time_get_cpu_seconds_( void );

/*--------------------------------------------------------------------------
 * With timing off
 *--------------------------------------------------------------------------*/

#ifndef HYPRE_TIMING

#define hypre_InitializeTiming(name) 0
#define hypre_FinalizeTiming(index)
#define hypre_IncFLOPCount(inc)
#define hypre_BeginTiming(i)
#define hypre_EndTiming(i)
#define hypre_PrintTiming(heading, comm)
#define hypre_ClearTiming()

/*--------------------------------------------------------------------------
 * With timing on
 *--------------------------------------------------------------------------*/

#else

/*-------------------------------------------------------
 * Global timing structure
 *-------------------------------------------------------*/

typedef struct
{
   HYPRE_Real  *wall_time;
   HYPRE_Real  *cpu_time;
   HYPRE_Real  *flops;
   char   **name;
   HYPRE_Int     *state;     /* boolean flag to allow for recursive timing */
   HYPRE_Int     *num_regs;  /* count of how many times a name is registered */

   HYPRE_Int      num_names;
   HYPRE_Int      size;

   HYPRE_Real   wall_count;
   HYPRE_Real   CPU_count;
   HYPRE_Real   FLOP_count;

} hypre_TimingType;

#ifdef HYPRE_TIMING_GLOBALS
hypre_TimingType *hypre_global_timing = NULL;
#else
extern hypre_TimingType *hypre_global_timing;
#endif

/*-------------------------------------------------------
 * Accessor functions
 *-------------------------------------------------------*/

#define hypre_TimingWallTime(i) (hypre_global_timing -> wall_time[(i)])
#define hypre_TimingCPUTime(i)  (hypre_global_timing -> cpu_time[(i)])
#define hypre_TimingFLOPS(i)    (hypre_global_timing -> flops[(i)])
#define hypre_TimingName(i)     (hypre_global_timing -> name[(i)])
#define hypre_TimingState(i)    (hypre_global_timing -> state[(i)])
#define hypre_TimingNumRegs(i)  (hypre_global_timing -> num_regs[(i)])
#define hypre_TimingWallCount   (hypre_global_timing -> wall_count)
#define hypre_TimingCPUCount    (hypre_global_timing -> CPU_count)
#define hypre_TimingFLOPCount   (hypre_global_timing -> FLOP_count)

/*-------------------------------------------------------
 * Prototypes
 *-------------------------------------------------------*/

/* timing.c */
HYPRE_Int hypre_InitializeTiming( const char *name );
HYPRE_Int hypre_FinalizeTiming( HYPRE_Int time_index );
HYPRE_Int hypre_IncFLOPCount( HYPRE_BigInt inc );
HYPRE_Int hypre_BeginTiming( HYPRE_Int time_index );
HYPRE_Int hypre_EndTiming( HYPRE_Int time_index );
HYPRE_Int hypre_ClearTiming( void );
HYPRE_Int hypre_PrintTiming( const char *heading , MPI_Comm comm );

#endif

#ifdef __cplusplus
}
#endif

#endif

/******************************************************************************
 * Copyright 1998-2019 Lawrence Livermore National Security, LLC and other
 * HYPRE Project Developers. See the top-level COPYRIGHT file for details.
 *
 * SPDX-License-Identifier: (Apache-2.0 OR MIT)
 ******************************************************************************/

/******************************************************************************
 *
 * Header file link lists
 *
 *****************************************************************************/

#ifndef HYPRE_LINKLIST_HEADER
#define HYPRE_LINKLIST_HEADER

#include <stdlib.h>
#include <stdio.h>
#include <string.h>

#ifdef __cplusplus
extern "C" {
#endif

struct double_linked_list
{
   HYPRE_Int                  data;
   struct double_linked_list *next_elt;
   struct double_linked_list *prev_elt;
   HYPRE_Int                  head;
   HYPRE_Int                  tail;
};

typedef struct double_linked_list hypre_ListElement;
typedef hypre_ListElement *hypre_LinkList;  

#ifdef __cplusplus
}
#endif

#endif

/******************************************************************************
 * Copyright 1998-2019 Lawrence Livermore National Security, LLC and other
 * HYPRE Project Developers. See the top-level COPYRIGHT file for details.
 *
 * SPDX-License-Identifier: (Apache-2.0 OR MIT)
 ******************************************************************************/

#ifndef hypre_EXCHANGE_DATA_HEADER
#define hypre_EXCHANGE_DATA_HEADER

#define hypre_BinaryTreeParentId(tree)      (tree->parent_id)
#define hypre_BinaryTreeNumChild(tree)      (tree->num_child)
#define hypre_BinaryTreeChildIds(tree)      (tree->child_id)
#define hypre_BinaryTreeChildId(tree, i)    (tree->child_id[i])

typedef struct
{
   HYPRE_Int                   parent_id;
   HYPRE_Int                   num_child;
   HYPRE_Int		        *child_id;
} hypre_BinaryTree;

/* In the fill_response() function the user needs to set the recv__buf
   and the response_message_size.  Memory of size send_response_storage has been
   alllocated for the send_buf (in exchange_data) - if more is needed, then
   realloc and adjust
   the send_response_storage.  The realloc amount should be storage+overhead. 
   If the response is an empty "confirmation" message, then set
   response_message_size =0 (and do not modify the send_buf) */

typedef struct
{
   HYPRE_Int    (*fill_response)(void* recv_buf, HYPRE_Int contact_size, 
                           HYPRE_Int contact_proc, void* response_obj, 
                           MPI_Comm comm, void** response_buf, 
                           HYPRE_Int* response_message_size);
   HYPRE_Int     send_response_overhead; /*set by exchange data */
   HYPRE_Int     send_response_storage;  /*storage allocated for send_response_buf*/
   void    *data1;                 /*data fields user may want to access in fill_response */
   void    *data2;
   
} hypre_DataExchangeResponse;

HYPRE_Int hypre_CreateBinaryTree(HYPRE_Int, HYPRE_Int, hypre_BinaryTree*);
HYPRE_Int hypre_DestroyBinaryTree(hypre_BinaryTree*);

HYPRE_Int hypre_DataExchangeList(HYPRE_Int num_contacts, 
		     HYPRE_Int *contact_proc_list, void *contact_send_buf, 
		     HYPRE_Int *contact_send_buf_starts, HYPRE_Int contact_obj_size, 
                     HYPRE_Int response_obj_size,
		     hypre_DataExchangeResponse *response_obj, HYPRE_Int max_response_size, 
                     HYPRE_Int rnum, MPI_Comm comm,  void **p_response_recv_buf, 
                     HYPRE_Int **p_response_recv_buf_starts);

#endif /* end of header */

/******************************************************************************
 * Copyright 1998-2019 Lawrence Livermore National Security, LLC and other
 * HYPRE Project Developers. See the top-level COPYRIGHT file for details.
 *
 * SPDX-License-Identifier: (Apache-2.0 OR MIT)
 ******************************************************************************/

#ifndef hypre_ERROR_HEADER
#define hypre_ERROR_HEADER

/*--------------------------------------------------------------------------
 * Global variable used in hypre error checking
 *--------------------------------------------------------------------------*/

extern HYPRE_Int hypre__global_error;
#define hypre_error_flag  hypre__global_error

/*--------------------------------------------------------------------------
 * HYPRE error macros
 *--------------------------------------------------------------------------*/

void hypre_error_handler(const char *filename, HYPRE_Int line, HYPRE_Int ierr, const char *msg);
#define hypre_error(IERR)  hypre_error_handler(__FILE__, __LINE__, IERR, NULL)
#define hypre_error_w_msg(IERR, msg)  hypre_error_handler(__FILE__, __LINE__, IERR, msg)
#define hypre_error_in_arg(IARG)  hypre_error(HYPRE_ERROR_ARG | IARG<<3)
#ifdef NDEBUG
#define hypre_assert(EX)
#else
#define hypre_assert(EX) if (!(EX)) {hypre_fprintf(stderr,"hypre_assert failed: %s\n", #EX); hypre_error(1);}
#endif

#endif

/******************************************************************************
 * Copyright 1998-2019 Lawrence Livermore National Security, LLC and other
 * HYPRE Project Developers. See the top-level COPYRIGHT file for details.
 *
 * SPDX-License-Identifier: (Apache-2.0 OR MIT)
 ******************************************************************************/

/******************************************************************************
 *
 * Header file for Caliper instrumentation macros
 *
 *****************************************************************************/

#ifndef CALIPER_INSTRUMENTATION_HEADER
#define CALIPER_INSTRUMENTATION_HEADER

#include "HYPRE_config.h"

#ifdef HYPRE_USING_CALIPER

#include <caliper/cali.h>

#define HYPRE_ANNOTATION_BEGIN( str ) cali_begin_string_byname("hypre.kernel", str)
#define HYPRE_ANNOTATION_END( str ) cali_end_byname("hypre.kernel")

#else

#define HYPRE_ANNOTATION_BEGIN( str ) 
#define HYPRE_ANNOTATION_END( str ) 

#endif

#endif /* CALIPER_INSTRUMENTATION_HEADER */

/******************************************************************************
 * Copyright 1998-2019 Lawrence Livermore National Security, LLC and other
 * HYPRE Project Developers. See the top-level COPYRIGHT file for details.
 *
 * SPDX-License-Identifier: (Apache-2.0 OR MIT)
 ******************************************************************************/

#ifndef HYPRE_CUDA_UTILS_H
#define HYPRE_CUDA_UTILS_H

#if defined(HYPRE_USING_CUDA) || defined(HYPRE_USING_DEVICE_OPENMP)

HYPRE_Int hypre_printf( const char *format , ... );

#ifdef __cplusplus
extern "C++" {
#endif

#include <cuda.h>
#include <cuda_runtime.h>
#include <assert.h>
#include <curand.h>
#include <cublas_v2.h>
#include <cusparse.h>

#ifndef CUDART_VERSION
#error CUDART_VERSION Undefined!
#endif

#ifndef CUDA_VERSION
#error CUDA_VERSION Undefined!
#endif

#if defined(HYPRE_USING_CUDA)
#include <thrust/execution_policy.h>
#include <thrust/system/cuda/execution_policy.h>
#include <thrust/count.h>
#include <thrust/device_ptr.h>
#include <thrust/unique.h>
#include <thrust/sort.h>
#include <thrust/binary_search.h>
#include <thrust/iterator/constant_iterator.h>
#include <thrust/iterator/counting_iterator.h>
#include <thrust/transform.h>
#include <thrust/functional.h>
#include <thrust/gather.h>
#include <thrust/scan.h>
#include <thrust/fill.h>
#include <thrust/adjacent_difference.h>
#include <thrust/inner_product.h>
using namespace thrust::placeholders;
#endif // #if defined(HYPRE_USING_CUDA)

#define HYPRE_WARP_SIZE       32
#define HYPRE_WARP_FULL_MASK  0xFFFFFFFF
#define HYPRE_MAX_NUM_WARPS   (64 * 64 * 32)
#define HYPRE_FLT_LARGE       1e30
#define HYPRE_1D_BLOCK_SIZE   512
#define HYPRE_MAX_NUM_STREAMS 10

/* - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - -
 * macro for launching CUDA kernels, CUDA, Thrust, Cusparse, Curand calls
 *                    NOTE: IN HYPRE'S DEFAULT STREAM
 * - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - -
 */

#define HYPRE_CUDA_LAUNCH(kernel_name, gridsize, blocksize, ...)                                                   \
{                                                                                                                  \
   if ( gridsize.x  == 0 || gridsize.y  == 0 || gridsize.z  == 0 ||                                                \
        blocksize.x == 0 || blocksize.y == 0 || blocksize.z == 0 )                                                 \
   {                                                                                                               \
      /* hypre_printf("Warning %s %d: Zero CUDA grid/block (%d %d %d) (%d %d %d)\n",                               \
                   __FILE__, __LINE__,                                                                             \
                   gridsize.x, gridsize.y, gridsize.z, blocksize.x, blocksize.y, blocksize.z); */                  \
   }                                                                                                               \
   else                                                                                                            \
   {                                                                                                               \
      (kernel_name) <<< (gridsize), (blocksize), 0, hypre_HandleCudaComputeStream(hypre_handle) >>> (__VA_ARGS__); \
   }                                                                                                               \
}


#define HYPRE_THRUST_CALL(func_name, ...)                                                    \
   thrust::func_name(                                                                        \
   thrust::cuda::par.on(hypre_HandleCudaComputeStream(hypre_handle)), __VA_ARGS__);          \

#define HYPRE_CUBLAS_CALL(call) do {                                                         \
   cublasStatus_t err = call;                                                                \
   if (CUBLAS_STATUS_SUCCESS != err) {                                                       \
      hypre_printf("CUBLAS ERROR (code = %d, %d) at %s:%d\n",                                \
            err, err == CUBLAS_STATUS_EXECUTION_FAILED, __FILE__, __LINE__);                 \
      exit(1);                                                                               \
   } } while(0)

#define HYPRE_CUSPARSE_CALL(call) do {                                                       \
   cusparseStatus_t err = call;                                                              \
   if (CUSPARSE_STATUS_SUCCESS != err) {                                                     \
      hypre_printf("CUSPARSE ERROR (code = %d, %d) at %s:%d\n",                              \
            err, err == CUSPARSE_STATUS_EXECUTION_FAILED, __FILE__, __LINE__);               \
      exit(1);                                                                               \
   } } while(0)


#define HYPRE_CURAND_CALL(call) do {                        \
   curandStatus_t err = call;                               \
   if (CURAND_STATUS_SUCCESS != err) {                      \
      hypre_printf("CURAND ERROR (code = %d) at %s:%d\n",   \
                   err, __FILE__, __LINE__);                \
      exit(1);                                              \
   } } while(0)


#define HYPRE_CUDA_CALL(call) do {                                                                             \
   cudaError_t err = call;                                                                                     \
   if (cudaSuccess != err) {                                                                                   \
      hypre_printf("CUDA ERROR (code = %d, %s) at %s:%d\n", err, cudaGetErrorString(err), __FILE__, __LINE__); \
      exit(1);                                                                                                 \
   } } while(0)

#if defined(HYPRE_USING_CUDA)
/* return the number of threads in block */
template <hypre_int dim>
static __device__ __forceinline__
hypre_int hypre_cuda_get_num_threads()
{
   switch (dim)
   {
      case 1:
         return (blockDim.x);
      case 2:
         return (blockDim.x * blockDim.y);
      case 3:
         return (blockDim.x * blockDim.y * blockDim.z);
   }

   return -1;
}

/* return the flattened thread id in block */
template <hypre_int dim>
static __device__ __forceinline__
hypre_int hypre_cuda_get_thread_id()
{
   switch (dim)
   {
      case 1:
         return (threadIdx.x);
      case 2:
         return (threadIdx.y * blockDim.x + threadIdx.x);
      case 3:
         return (threadIdx.z * blockDim.x * blockDim.y + threadIdx.y * blockDim.x +
                 threadIdx.x);
   }

   return -1;
}

/* return the number of warps in block */
template <hypre_int dim>
static __device__ __forceinline__
hypre_int hypre_cuda_get_num_warps()
{
   return hypre_cuda_get_num_threads<dim>() >> 5;
}

/* return the warp id in block */
template <hypre_int dim>
static __device__ __forceinline__
hypre_int hypre_cuda_get_warp_id()
{
   return hypre_cuda_get_thread_id<dim>() >> 5;
}

/* return the thread lane id in warp */
template <hypre_int dim>
static __device__ __forceinline__
hypre_int hypre_cuda_get_lane_id()
{
   return hypre_cuda_get_thread_id<dim>() & (HYPRE_WARP_SIZE-1);
}

/* return the num of blocks in grid */
template <hypre_int dim>
static __device__ __forceinline__
hypre_int hypre_cuda_get_num_blocks()
{
   switch (dim)
   {
      case 1:
         return (gridDim.x);
      case 2:
         return (gridDim.x * gridDim.y);
      case 3:
         return (gridDim.x * gridDim.y * gridDim.z);
   }

   return -1;
}

/* return the flattened block id in grid */
template <hypre_int dim>
static __device__ __forceinline__
hypre_int hypre_cuda_get_block_id()
{
   switch (dim)
   {
      case 1:
         return (blockIdx.x);
      case 2:
         return (blockIdx.y * gridDim.x + blockIdx.x);
      case 3:
         return (blockIdx.z * gridDim.x * gridDim.y + blockIdx.y * gridDim.x +
                 blockIdx.x);
   }

   return -1;
}

/* return the number of threads in grid */
template <hypre_int bdim, hypre_int gdim>
static __device__ __forceinline__
hypre_int hypre_cuda_get_grid_num_threads()
{
   return hypre_cuda_get_num_blocks<gdim>() * hypre_cuda_get_num_threads<bdim>();
}

/* return the flattened thread id in grid */
template <hypre_int bdim, hypre_int gdim>
static __device__ __forceinline__
hypre_int hypre_cuda_get_grid_thread_id()
{
   return hypre_cuda_get_block_id<gdim>() * hypre_cuda_get_num_threads<bdim>() +
          hypre_cuda_get_thread_id<bdim>();
}

/* return the number of warps in grid */
template <hypre_int bdim, hypre_int gdim>
static __device__ __forceinline__
hypre_int hypre_cuda_get_grid_num_warps()
{
   return hypre_cuda_get_num_blocks<gdim>() * hypre_cuda_get_num_warps<bdim>();
}

/* return the flattened warp id in grid */
template <hypre_int bdim, hypre_int gdim>
static __device__ __forceinline__
hypre_int hypre_cuda_get_grid_warp_id()
{
   return hypre_cuda_get_block_id<gdim>() * hypre_cuda_get_num_warps<bdim>() +
          hypre_cuda_get_warp_id<bdim>();
}

#if CUDA_VERSION < 9000

template <typename T>
static __device__ __forceinline__
T __shfl_sync(unsigned mask, T val, hypre_int src_line, hypre_int width=32)
{
   return __shfl(val, src_line, width);
}

template <typename T>
static __device__ __forceinline__
T __shfl_down_sync(unsigned mask, T val, unsigned delta, hypre_int width=32)
{
   return __shfl_down(val, delta, width);
}

template <typename T>
static __device__ __forceinline__
T __shfl_xor_sync(unsigned mask, T val, unsigned lanemask, hypre_int width=32)
{
   return __shfl_xor(val, lanemask, width);
}

template <typename T>
static __device__ __forceinline__
T __shfl_up_sync(unsigned mask, T val, unsigned delta, hypre_int width=32)
{
   return __shfl_up(val, delta, width);
}

static __device__ __forceinline__
void __syncwarp()
{
}

#endif

template <typename T>
static __device__ __forceinline__
T read_only_load( const T *ptr )
{
   return __ldg( ptr );
}

template <typename T>
static __device__ __forceinline__
T warp_prefix_sum(hypre_int lane_id, T in, T &all_sum)
{
#pragma unroll
   for (hypre_int d = 2; d <= 32; d <<= 1)
   {
      T t = __shfl_up_sync(HYPRE_WARP_FULL_MASK, in, d >> 1);
      if ( (lane_id & (d - 1)) == d - 1 )
      {
         in += t;
      }
   }

   all_sum = __shfl_sync(HYPRE_WARP_FULL_MASK, in, 31);

   if (lane_id == 31)
   {
      in = 0;
   }

#pragma unroll
   for (hypre_int d = 16; d > 0; d >>= 1)
   {
      T t = __shfl_xor_sync(HYPRE_WARP_FULL_MASK, in, d);

      if ( (lane_id & (d - 1)) == d - 1)
      {
         if ( (lane_id & (d << 1 - 1)) == (d << 1 - 1) )
         {
            in += t;
         }
         else
         {
            in = t;
         }
      }
   }
   return in;
}

template <typename T>
static __device__ __forceinline__
T warp_reduce_sum(T in)
{
#pragma unroll
  for (hypre_int d = 16; d > 0; d >>= 1)
  {
    in += __shfl_down_sync(HYPRE_WARP_FULL_MASK, in, d);
  }
  return in;
}

template <typename T>
static __device__ __forceinline__
T warp_allreduce_sum(T in)
{
#pragma unroll
  for (hypre_int d = 16; d > 0; d >>= 1)
  {
    in += __shfl_xor_sync(HYPRE_WARP_FULL_MASK, in, d);
  }
  return in;
}

template <typename T>
static __device__ __forceinline__
T warp_reduce_max(T in)
{
#pragma unroll
  for (hypre_int d = 16; d > 0; d >>= 1)
  {
    in = max(in, __shfl_down_sync(HYPRE_WARP_FULL_MASK, in, d));
  }
  return in;
}

template <typename T>
static __device__ __forceinline__
T warp_allreduce_max(T in)
{
#pragma unroll
  for (hypre_int d = 16; d > 0; d >>= 1)
  {
    in = max(in, __shfl_xor_sync(HYPRE_WARP_FULL_MASK, in, d));
  }
  return in;
}

template <typename T>
static __device__ __forceinline__
T warp_reduce_min(T in)
{
#pragma unroll
  for (hypre_int d = 16; d > 0; d >>= 1)
  {
    in = min(in, __shfl_down_sync(HYPRE_WARP_FULL_MASK, in, d));
  }
  return in;
}

template <typename T>
static __device__ __forceinline__
T warp_allreduce_min(T in)
{
#pragma unroll
  for (hypre_int d = 16; d > 0; d >>= 1)
  {
    in = min(in, __shfl_xor_sync(HYPRE_WARP_FULL_MASK, in, d));
  }
  return in;
}

static __device__ __forceinline__
hypre_int next_power_of_2(hypre_int n)
{
   if (n <= 0)
   {
      return 0;
   }

   /* if n is power of 2, return itself */
   if ( (n & (n - 1)) == 0 )
   {
      return n;
   }

   n |= (n >>  1);
   n |= (n >>  2);
   n |= (n >>  4);
   n |= (n >>  8);
   n |= (n >> 16);
   n ^= (n >>  1);
   n  = (n <<  1);

   return n;
}

template<typename T>
struct absolute_value : public thrust::unary_function<T,T>
{
  __host__ __device__ T operator()(const T &x) const
  {
    return x < T(0) ? -x : x;
  }
};


template<typename T1, typename T2>
struct TupleComp1
{
   typedef thrust::tuple<T1, T2> Tuple;

   __host__ __device__ bool operator()(const Tuple& t1, const Tuple& t2)
   {
      if (thrust::get<0>(t1) < thrust::get<0>(t2))
      {
         return true;
      }
      if (thrust::get<0>(t1) > thrust::get<0>(t2))
      {
         return false;
      }
      return thrust::get<1>(t1) < thrust::get<1>(t2);
   }
};


template<typename T1, typename T2>
struct TupleComp2
{
   typedef thrust::tuple<T1, T2> Tuple;

   __host__ __device__ bool operator()(const Tuple& t1, const Tuple& t2)
   {
      if (thrust::get<0>(t1) < thrust::get<0>(t2))
      {
         return true;
      }
      if (thrust::get<0>(t1) > thrust::get<0>(t2))
      {
         return false;
      }
      return hypre_abs(thrust::get<1>(t1)) > hypre_abs(thrust::get<1>(t2));
   }
};

#endif // #if defined(HYPRE_USING_CUDA)

#ifdef __cplusplus
}
#endif

struct is_negative
{
   __host__ __device__ bool operator()(const HYPRE_Int &x)
   {
      return (x < 0);
   }
};

struct is_nonnegative
{
   __host__ __device__ bool operator()(const HYPRE_Int &x)
   {
      return (x >= 0);
   }
};

struct in_range
{
   HYPRE_Int low, up;

   in_range(HYPRE_Int low_, HYPRE_Int up_) { low = low_; up = up_; }

   __host__ __device__ bool operator()(const HYPRE_Int &x)
   {
      return (x >= low && x <= up);
   }
};

struct out_of_range
{
   HYPRE_Int low, up;

   out_of_range(HYPRE_Int low_, HYPRE_Int up_) { low = low_; up = up_; }

   __host__ __device__ bool operator()(const HYPRE_Int &x)
   {
      return (x < low || x > up);
   }
};

struct less_than
{
   HYPRE_Int val;

   less_than(HYPRE_Int val_) { val = val_; }

   __host__ __device__ bool operator()(const HYPRE_Int &x)
   {
      return (x < val);
   }
};


#if defined(HYPRE_USING_CUDA)
/* for struct solvers */
#define HYPRE_MIN_GPU_SIZE (131072)
extern HYPRE_Int hypre_exec_policy;
#define hypre_SetDeviceOn()  hypre_exec_policy = HYPRE_MEMORY_DEVICE
#define hypre_SetDeviceOff() hypre_exec_policy = HYPRE_MEMORY_HOST
#endif

#endif /* HYPRE_USING_CUDA */
#endif /* #ifndef HYPRE_CUDA_UTILS_H */

/******************************************************************************
 * Copyright 1998-2019 Lawrence Livermore National Security, LLC and other
 * HYPRE Project Developers. See the top-level COPYRIGHT file for details.
 *
 * SPDX-License-Identifier: (Apache-2.0 OR MIT)
 ******************************************************************************/

/******************************************************************************
 *
 * General structures and values
 *
 *****************************************************************************/

#ifndef HYPRE_HANDLE_H
#define HYPRE_HANDLE_H

#ifdef __cplusplus
extern "C++" {
#endif

#if defined(HYPRE_USING_CUDA) || defined(HYPRE_USING_DEVICE_OPENMP)
#include <vector>
#endif

typedef struct
{
   HYPRE_Int hypre_error;
#if defined(HYPRE_USING_CUDA) || defined(HYPRE_USING_DEVICE_OPENMP)
   HYPRE_Int default_exec_policy;
   HYPRE_Int cuda_device;
   /* by default, hypre puts GPU computations in this stream
    * Do not be confused with the default (null) CUDA stream */
   HYPRE_Int cuda_compute_stream_num;
   HYPRE_Int cuda_prefetch_stream_num;
   HYPRE_Int cuda_compute_stream_sync_default;
   std::vector<HYPRE_Int> cuda_compute_stream_sync;
   curandGenerator_t curand_gen;
   cublasHandle_t cublas_handle;
   cusparseHandle_t cusparse_handle;
   cusparseMatDescr_t cusparse_mat_descr;
   cudaStream_t cuda_streams[HYPRE_MAX_NUM_STREAMS];
   /* work space for hypre's CUDA reducer */
   void* cuda_reduce_buffer;
   /* device spgemm options */
   HYPRE_Int spgemm_use_cusparse;
   HYPRE_Int spgemm_num_passes;
   HYPRE_Int spgemm_rownnz_estimate_method;
   HYPRE_Int spgemm_rownnz_estimate_nsamples;
   float     spgemm_rownnz_estimate_mult_factor;
   char      spgemm_hash_type;
#endif
} hypre_Handle;

extern hypre_Handle *hypre_handle;

hypre_Handle* hypre_HandleCreate();
HYPRE_Int hypre_HandleDestroy(hypre_Handle *hypre_handle_);

/* accessor inline function to hypre_device_csr_handle */

#if defined(HYPRE_USING_CUDA) || defined(HYPRE_USING_DEVICE_OPENMP)
static inline HYPRE_Int &
hypre_HandleDefaultExecPolicy(hypre_Handle *hypre_handle_)
{
   return hypre_handle_->default_exec_policy;
}

static inline HYPRE_Int &
hypre_HandleCudaDevice(hypre_Handle *hypre_handle_)
{
   return hypre_handle_->cuda_device;
}

static inline HYPRE_Int &
hypre_HandleCudaComputeStreamNum(hypre_Handle *hypre_handle_)
{
   return hypre_handle_->cuda_compute_stream_num;
}

static inline HYPRE_Int &
hypre_HandleCudaPrefetchStreamNum(hypre_Handle *hypre_handle_)
{
   return hypre_handle_->cuda_prefetch_stream_num;
}

static inline HYPRE_Int &
hypre_HandleCudaComputeStreamSyncDefault(hypre_Handle *hypre_handle_)
{
   return hypre_handle_->cuda_compute_stream_sync_default;
}

static inline std::vector<HYPRE_Int> &
hypre_HandleCudaComputeStreamSync(hypre_Handle *hypre_handle_)
{
   return hypre_handle_->cuda_compute_stream_sync;
}

static inline cudaStream_t
hypre_HandleCudaStream(hypre_Handle *hypre_handle_, HYPRE_Int i)
{
   cudaStream_t stream = 0;
#if defined(HYPRE_USING_CUDA_STREAMS)
   if (i >= HYPRE_MAX_NUM_STREAMS)
   {
      /* return the default stream, i.e., the NULL stream */
      /*
      hypre_printf("CUDA stream %d exceeds the max number %d\n",
                   i, HYPRE_MAX_NUM_STREAMS);
      */
      return NULL;
   }

   if (hypre_handle_->cuda_streams[i])
   {
      return hypre_handle_->cuda_streams[i];
   }

   //HYPRE_CUDA_CALL(cudaStreamCreateWithFlags(&stream,cudaStreamNonBlocking));
   HYPRE_CUDA_CALL(cudaStreamCreateWithFlags(&stream, cudaStreamDefault));

   hypre_handle_->cuda_streams[i] = stream;
#endif

   return stream;
}

static inline cudaStream_t
hypre_HandleCudaComputeStream(hypre_Handle *hypre_handle_)
{
   return hypre_HandleCudaStream(hypre_handle_,
                                 hypre_HandleCudaComputeStreamNum(hypre_handle_));
}

static inline cudaStream_t
hypre_HandleCudaPrefetchStream(hypre_Handle *hypre_handle_)
{
   return hypre_HandleCudaStream(hypre_handle_,
                                 hypre_HandleCudaPrefetchStreamNum(hypre_handle_));
}

static inline curandGenerator_t
hypre_HandleCurandGenerator(hypre_Handle *hypre_handle_)
{
   if (hypre_handle_->curand_gen)
   {
      return hypre_handle_->curand_gen;
   }

   curandGenerator_t gen;
   HYPRE_CURAND_CALL( curandCreateGenerator(&gen, CURAND_RNG_PSEUDO_DEFAULT) );
   HYPRE_CURAND_CALL( curandSetPseudoRandomGeneratorSeed(gen, 1234ULL) );
   HYPRE_CURAND_CALL( curandSetStream(gen, hypre_HandleCudaComputeStream(hypre_handle_)) );

   hypre_handle_->curand_gen = gen;

   return gen;
}

static inline cublasHandle_t
hypre_HandleCublasHandle(hypre_Handle *hypre_handle_)
{
   if (hypre_handle_->cublas_handle)
   {
      return hypre_handle_->cublas_handle;
   }

   cublasHandle_t handle;
   HYPRE_CUBLAS_CALL( cublasCreate(&handle) );

   HYPRE_CUBLAS_CALL( cublasSetStream(handle, hypre_HandleCudaComputeStream(hypre_handle_)) );

   hypre_handle_->cublas_handle = handle;

   return handle;
}

static inline cusparseHandle_t
hypre_HandleCusparseHandle(hypre_Handle *hypre_handle_)
{
   if (hypre_handle_->cusparse_handle)
   {
      return hypre_handle_->cusparse_handle;
   }

   cusparseHandle_t handle;
   HYPRE_CUSPARSE_CALL( cusparseCreate(&handle) );

   HYPRE_CUSPARSE_CALL( cusparseSetStream(handle, hypre_HandleCudaComputeStream(hypre_handle_)) );

   hypre_handle_->cusparse_handle = handle;

   return handle;
}

static inline cusparseMatDescr_t
hypre_HandleCusparseMatDescr(hypre_Handle *hypre_handle_)
{
   if (hypre_handle_->cusparse_mat_descr)
   {
      return hypre_handle_->cusparse_mat_descr;
   }

   cusparseMatDescr_t mat_descr;
   HYPRE_CUSPARSE_CALL( cusparseCreateMatDescr(&mat_descr) );
   HYPRE_CUSPARSE_CALL( cusparseSetMatType(mat_descr, CUSPARSE_MATRIX_TYPE_GENERAL) );
   HYPRE_CUSPARSE_CALL( cusparseSetMatIndexBase(mat_descr, CUSPARSE_INDEX_BASE_ZERO) );

   hypre_handle_->cusparse_mat_descr = mat_descr;

   return mat_descr;
}

#endif /* defined(HYPRE_USING_CUDA) || defined(HYPRE_USING_DEVICE_OPENMP) */

static inline void
hypre_HandleCudaComputeStreamSyncPush(hypre_Handle *hypre_handle_, HYPRE_Int sync)
{
#if defined(HYPRE_USING_CUDA) && defined(HYPRE_USING_UNIFIED_MEMORY)
   hypre_HandleCudaComputeStreamSync(hypre_handle_).push_back(sync);
#endif
}

static inline void
hypre_HandleCudaComputeStreamSyncPop(hypre_Handle *hypre_handle_)
{
#if defined(HYPRE_USING_CUDA) && defined(HYPRE_USING_UNIFIED_MEMORY)
   hypre_HandleCudaComputeStreamSync(hypre_handle_).pop_back();
#endif
}

/* synchronize the default stream */
static inline HYPRE_Int
hypre_SyncCudaComputeStream(hypre_Handle *hypre_handle_)
{
#if defined(HYPRE_USING_UNIFIED_MEMORY)
#if defined(HYPRE_USING_CUDA)
   assert(!hypre_HandleCudaComputeStreamSync(hypre_handle_).empty());

   if ( hypre_HandleCudaComputeStreamSync(hypre_handle_).back() )
   {
      HYPRE_CUDA_CALL( cudaStreamSynchronize(hypre_HandleCudaComputeStream(hypre_handle_)) );
   }
#endif
#if defined(HYPRE_USING_DEVICE_OPENMP)
   HYPRE_CUDA_CALL( cudaDeviceSynchronize() );
#endif
#endif /* #if defined(HYPRE_USING_UNIFIED_MEMORY) */
   return hypre_error_flag;
}

#ifdef __cplusplus
}
#endif

#endif

/******************************************************************************
 * Copyright 1998-2019 Lawrence Livermore National Security, LLC and other
 * HYPRE Project Developers. See the top-level COPYRIGHT file for details.
 *
 * SPDX-License-Identifier: (Apache-2.0 OR MIT)
 ******************************************************************************/

#ifndef HYPRE_NVTX_H
#define HYPRE_NVTX_H

#ifdef HYPRE_USING_NVTX

#include "nvToolsExt.h"
#include "nvToolsExtCudaRt.h"

static const uint32_t colors[] = { 0x0000ff00, 0x000000ff, 0x00ffff00, 0x00ff00ff, 0x0000ffff, 0x00ff0000, 0x00ffffff };
static const hypre_int num_colors = sizeof(colors)/sizeof(uint32_t);

#define PUSH_RANGE(name,cid) { \
    hypre_int color_id = cid; \
    color_id = color_id%num_colors;\
    nvtxEventAttributes_t eventAttrib = {0}; \
    eventAttrib.version = NVTX_VERSION; \
    eventAttrib.size = NVTX_EVENT_ATTRIB_STRUCT_SIZE; \
    eventAttrib.colorType = NVTX_COLOR_ARGB; \
    eventAttrib.color = colors[color_id]; \
    eventAttrib.messageType = NVTX_MESSAGE_TYPE_ASCII; \
    eventAttrib.message.ascii = name; \
    nvtxRangePushEx(&eventAttrib); \
}

#define POP_RANGE nvtxRangePop();

#else /* HYPRE_USING_NVTX */

#define PUSH_RANGE(name,cid)
#define POP_RANGE

#endif /* HYPRE_USING_NVTX */

#endif /* HYPRE_NVTX_H */
/******************************************************************************
 * Copyright 1998-2019 Lawrence Livermore National Security, LLC and other
 * HYPRE Project Developers. See the top-level COPYRIGHT file for details.
 *
 * SPDX-License-Identifier: (Apache-2.0 OR MIT)
 ******************************************************************************/

/* CUDA reducer class */

#ifndef HYPRE_CUDA_REDUCER_H
#define HYPRE_CUDA_REDUCER_H

#if defined(HYPRE_USING_CUDA)
#if !defined(HYPRE_USING_RAJA) && !defined(HYPRE_USING_KOKKOS)

#ifdef __cplusplus
extern "C++" {
#endif

template<typename T> void OneBlockReduce(T *d_arr, HYPRE_Int N, T *h_out);

struct HYPRE_double4
{
   HYPRE_Real x,y,z,w;

   __host__ __device__
   HYPRE_double4() {}

   __host__ __device__
   HYPRE_double4(HYPRE_Real x1, HYPRE_Real x2, HYPRE_Real x3, HYPRE_Real x4)
   {
      x = x1;
      y = x2;
      z = x3;
      w = x4;
   }

   __host__ __device__
   void operator=(HYPRE_Real val)
   {
      x = y = z = w = val;
   }

   __host__ __device__
   void operator+=(HYPRE_double4 rhs)
   {
      x += rhs.x;
      y += rhs.y;
      z += rhs.z;
      w += rhs.w;
   }

};

struct HYPRE_double6
{
   HYPRE_Real x,y,z,w,u,v;

   __host__ __device__
   HYPRE_double6() {}

   __host__ __device__
   HYPRE_double6(HYPRE_Real x1, HYPRE_Real x2, HYPRE_Real x3, HYPRE_Real x4,
                 HYPRE_Real x5, HYPRE_Real x6)
   {
      x = x1;
      y = x2;
      z = x3;
      w = x4;
      u = x5;
      v = x6;
   }

   __host__ __device__
   void operator=(HYPRE_Real val)
   {
      x = y = z = w = u = v = val;
   }

   __host__ __device__
   void operator+=(HYPRE_double6 rhs)
   {
      x += rhs.x;
      y += rhs.y;
      z += rhs.z;
      w += rhs.w;
      u += rhs.u;
      v += rhs.v;
   }

};

/* reduction within a warp */
__inline__ __host__ __device__
HYPRE_Real warpReduceSum(HYPRE_Real val)
{
#ifdef __CUDA_ARCH__
  for (HYPRE_Int offset = warpSize/2; offset > 0; offset /= 2)
  {
    val += __shfl_down_sync(HYPRE_WARP_FULL_MASK, val, offset);
  }
#endif
  return val;
}

__inline__ __host__ __device__
HYPRE_double4 warpReduceSum(HYPRE_double4 val) {
#ifdef __CUDA_ARCH__
  for (HYPRE_Int offset = warpSize / 2; offset > 0; offset /= 2)
  {
    val.x += __shfl_down_sync(HYPRE_WARP_FULL_MASK, val.x, offset);
    val.y += __shfl_down_sync(HYPRE_WARP_FULL_MASK, val.y, offset);
    val.z += __shfl_down_sync(HYPRE_WARP_FULL_MASK, val.z, offset);
    val.w += __shfl_down_sync(HYPRE_WARP_FULL_MASK, val.w, offset);
  }
#endif
  return val;
}

__inline__ __host__ __device__
HYPRE_double6 warpReduceSum(HYPRE_double6 val) {
#ifdef __CUDA_ARCH__
  for (HYPRE_Int offset = warpSize / 2; offset > 0; offset /= 2)
  {
    val.x += __shfl_down_sync(HYPRE_WARP_FULL_MASK, val.x, offset);
    val.y += __shfl_down_sync(HYPRE_WARP_FULL_MASK, val.y, offset);
    val.z += __shfl_down_sync(HYPRE_WARP_FULL_MASK, val.z, offset);
    val.w += __shfl_down_sync(HYPRE_WARP_FULL_MASK, val.w, offset);
    val.u += __shfl_down_sync(HYPRE_WARP_FULL_MASK, val.u, offset);
    val.v += __shfl_down_sync(HYPRE_WARP_FULL_MASK, val.v, offset);
  }
#endif
  return val;
}

/* reduction within a block */
template <typename T>
__inline__ __host__ __device__
T blockReduceSum(T val)
{
#ifdef __CUDA_ARCH__
   //static __shared__ T shared[32]; // Shared mem for 32 partial sums
   __shared__ T shared[32];        // Shared mem for 32 partial sums
   //HYPRE_Int lane = threadIdx.x % warpSize;
   //HYPRE_Int wid  = threadIdx.x / warpSize;
   HYPRE_Int lane = threadIdx.x & (warpSize - 1);
   HYPRE_Int wid  = threadIdx.x >> 5;

   val = warpReduceSum(val);       // Each warp performs partial reduction

   if (lane == 0)
   {
      shared[wid] = val;          // Write reduced value to shared memory
   }

   __syncthreads();               // Wait for all partial reductions

   //read from shared memory only if that warp existed
   if (threadIdx.x < blockDim.x / warpSize)
   {
      val = shared[lane];
   }
   else
   {
      val = 0.0;
   }

   if (wid == 0)
   {
      val = warpReduceSum(val); //Final reduce within first warp
   }

#endif
   return val;
}

template<typename T>
__global__ void
OneBlockReduceKernel(T *arr, HYPRE_Int N)
{
   T sum;
   sum = 0.0;
   if (threadIdx.x < N)
   {
      sum = arr[threadIdx.x];
   }
   sum = blockReduceSum(sum);
   if (threadIdx.x == 0)
   {
      arr[0] = sum;
   }
}

/* Reducer class */
template <typename T>
struct ReduceSum
{
   T init;                    /* initial value passed in */
   mutable T __thread_sum;    /* place to hold local sum of a thread,
                                 and partial sum of a block */
   T *d_buf;                  /* place to store partial sum within blocks
                                 in the 1st round, used in the 2nd round */
   HYPRE_Int nblocks;         /* number of blocks used in the 1st round */

   /* constructor
    * val is the initial value (added to the reduced sum) */
   __host__
   ReduceSum(T val)
   {
      init = val;
      __thread_sum = 0.0;
      nblocks = -1;

      if (hypre_handle->cuda_reduce_buffer == NULL)
      {
         /* allocate for the max size for reducing double6 type */
         hypre_handle->cuda_reduce_buffer =
            hypre_TAlloc(HYPRE_double6, 1024, HYPRE_MEMORY_DEVICE);
      }

      d_buf = (T*) hypre_handle->cuda_reduce_buffer;
   }

   /* copy constructor */
   __host__ __device__
   ReduceSum(const ReduceSum<T>& other)
   {
      *this = other;
   }

   /* reduction within blocks */
   __host__ __device__
   void BlockReduce() const
   {
#ifdef __CUDA_ARCH__
      __thread_sum = blockReduceSum(__thread_sum);
      if (threadIdx.x == 0)
      {
         d_buf[blockIdx.x] = __thread_sum;
      }
#endif
   }

   __host__ __device__
   void operator+=(T val) const
   {
      __thread_sum += val;
   }

   /* invoke the 2nd reduction at the time want the sum from the reducer */
   __host__
   operator T()
   {
      T val;
      /* 2nd reduction with only *one* block */
      assert(nblocks >= 0 && nblocks <= 1024);
      const dim3 gDim(1), bDim(1024);
      HYPRE_CUDA_LAUNCH( OneBlockReduceKernel, gDim, bDim, d_buf, nblocks );
      hypre_TMemcpy(&val, d_buf, T, 1, HYPRE_MEMORY_HOST, HYPRE_MEMORY_DEVICE);
      val += init;

      return val;
   }

   /* destructor */
   __host__ __device__
   ~ReduceSum<T>()
   {
   }
};

#ifdef __cplusplus
}
#endif

#endif /* #if !defined(HYPRE_USING_RAJA) && !defined(HYPRE_USING_KOKKOS) */
#endif /* #if defined(HYPRE_USING_CUDA) */
#endif /* #ifndef HYPRE_CUDA_REDUCER_H */

/******************************************************************************
 * Copyright 1998-2019 Lawrence Livermore National Security, LLC and other
 * HYPRE Project Developers. See the top-level COPYRIGHT file for details.
 *
 * SPDX-License-Identifier: (Apache-2.0 OR MIT)
 ******************************************************************************/

/* amg_linklist.c */
void hypre_dispose_elt ( hypre_LinkList element_ptr );
void hypre_remove_point ( hypre_LinkList *LoL_head_ptr , hypre_LinkList *LoL_tail_ptr , HYPRE_Int measure , HYPRE_Int index , HYPRE_Int *lists , HYPRE_Int *where );
hypre_LinkList hypre_create_elt ( HYPRE_Int Item );
void hypre_enter_on_lists ( hypre_LinkList *LoL_head_ptr , hypre_LinkList *LoL_tail_ptr , HYPRE_Int measure , HYPRE_Int index , HYPRE_Int *lists , HYPRE_Int *where );

/* binsearch.c */
HYPRE_Int hypre_BinarySearch ( HYPRE_Int *list , HYPRE_Int value , HYPRE_Int list_length );
HYPRE_Int hypre_BigBinarySearch(HYPRE_BigInt *list, HYPRE_BigInt value, HYPRE_Int list_length);
HYPRE_Int hypre_BinarySearch2 ( HYPRE_Int *list , HYPRE_Int value , HYPRE_Int low , HYPRE_Int high , HYPRE_Int *spot );
HYPRE_Int *hypre_LowerBound( HYPRE_Int *first, HYPRE_Int *last, HYPRE_Int value );
HYPRE_BigInt *hypre_BigLowerBound( HYPRE_BigInt *first, HYPRE_BigInt *last, HYPRE_BigInt value );

/* hypre_complex.c */
#ifdef HYPRE_COMPLEX
HYPRE_Complex hypre_conj( HYPRE_Complex value );
HYPRE_Real    hypre_cabs( HYPRE_Complex value );
HYPRE_Real    hypre_creal( HYPRE_Complex value );
HYPRE_Real    hypre_cimag( HYPRE_Complex value );
#else
#define hypre_conj(value)  value
#define hypre_cabs(value)  fabs(value)
#define hypre_creal(value) value
#define hypre_cimag(value) 0.0
#endif

/* hypre_general.c */
HYPRE_Int HYPRE_Init( hypre_int argc, char *argv[] );
HYPRE_Int HYPRE_Finalize();
HYPRE_Int hypre_GetDevice(hypre_Handle *hypre_handle);
HYPRE_Int hypre_SetDevice(HYPRE_Int use_device, hypre_Handle *hypre_handle);
HYPRE_Int hypre_SyncCudaDefaultStream(hypre_Handle *hypre_handle);
void hypre_SetExecPolicy( HYPRE_Int policy );
HYPRE_Int hypre_GetExecPolicy1(HYPRE_Int location);
HYPRE_Int hypre_GetExecPolicy2(HYPRE_Int location1, HYPRE_Int location2);

/* hypre_printf.c */
// #ifdef HYPRE_BIGINT
HYPRE_Int hypre_ndigits( HYPRE_BigInt number );
HYPRE_Int hypre_printf( const char *format , ... );
HYPRE_Int hypre_fprintf( FILE *stream , const char *format, ... );
HYPRE_Int hypre_sprintf( char *s , const char *format, ... );
HYPRE_Int hypre_scanf( const char *format , ... );
HYPRE_Int hypre_fscanf( FILE *stream , const char *format, ... );
HYPRE_Int hypre_sscanf( char *s , const char *format, ... );
// #else
// #define hypre_printf  printf
// #define hypre_fprintf fprintf
// #define hypre_sprintf sprintf
// #define hypre_scanf   scanf
// #define hypre_fscanf  fscanf
// #define hypre_sscanf  sscanf
// #endif

/* hypre_qsort.c */
void hypre_swap ( HYPRE_Int *v , HYPRE_Int i , HYPRE_Int j );
void hypre_swap2 ( HYPRE_Int *v , HYPRE_Real *w , HYPRE_Int i , HYPRE_Int j );
void hypre_BigSwap2 ( HYPRE_BigInt *v , HYPRE_Real *w , HYPRE_Int i , HYPRE_Int j );
void hypre_swap2i ( HYPRE_Int *v , HYPRE_Int *w , HYPRE_Int i , HYPRE_Int j );
void hypre_swap3i ( HYPRE_Int *v , HYPRE_Int *w , HYPRE_Int *z , HYPRE_Int i , HYPRE_Int j );
void hypre_swap3_d ( HYPRE_Real *v , HYPRE_Int *w , HYPRE_Int *z , HYPRE_Int i , HYPRE_Int j );
void hypre_BigSwap4_d ( HYPRE_Real *v , HYPRE_BigInt *w , HYPRE_Int *z , HYPRE_Int *y , HYPRE_Int i , HYPRE_Int j );
void hypre_swap_d ( HYPRE_Real *v , HYPRE_Int i , HYPRE_Int j );
void hypre_qsort0 ( HYPRE_Int *v , HYPRE_Int left , HYPRE_Int right );
void hypre_qsort1 ( HYPRE_Int *v , HYPRE_Real *w , HYPRE_Int left , HYPRE_Int right );
void hypre_BigQsort1 ( HYPRE_BigInt *v , HYPRE_Real *w , HYPRE_Int left , HYPRE_Int right );
void hypre_qsort2i ( HYPRE_Int *v , HYPRE_Int *w , HYPRE_Int left , HYPRE_Int right );
void hypre_qsort2 ( HYPRE_Int *v , HYPRE_Real *w , HYPRE_Int left , HYPRE_Int right );
void hypre_qsort2abs( HYPRE_Int *v, HYPRE_Real *w, HYPRE_Int left, HYPRE_Int right );
void hypre_qsort3i ( HYPRE_Int *v , HYPRE_Int *w , HYPRE_Int *z , HYPRE_Int left , HYPRE_Int right );
void hypre_qsort3_abs ( HYPRE_Real *v , HYPRE_Int *w , HYPRE_Int *z , HYPRE_Int left , HYPRE_Int right );
void hypre_BigQsort4_abs ( HYPRE_Real *v , HYPRE_BigInt *w , HYPRE_Int *z , HYPRE_Int *y , HYPRE_Int left , HYPRE_Int right );
void hypre_qsort_abs ( HYPRE_Real *w , HYPRE_Int left , HYPRE_Int right );
void hypre_BigSwapbi(HYPRE_BigInt  *v, HYPRE_Int  *w, HYPRE_Int  i, HYPRE_Int  j );
void hypre_BigQsortbi( HYPRE_BigInt *v, HYPRE_Int *w, HYPRE_Int  left, HYPRE_Int  right );
void hypre_BigSwapLoc(HYPRE_BigInt  *v, HYPRE_Int  *w, HYPRE_Int  i, HYPRE_Int  j );
void hypre_BigQsortbLoc( HYPRE_BigInt *v, HYPRE_Int *w, HYPRE_Int  left, HYPRE_Int  right );
void hypre_BigSwapb2i(HYPRE_BigInt  *v, HYPRE_Int  *w, HYPRE_Int  *z, HYPRE_Int  i, HYPRE_Int  j );
void hypre_BigQsortb2i( HYPRE_BigInt *v, HYPRE_Int *w, HYPRE_Int *z, HYPRE_Int  left, HYPRE_Int  right );
void hypre_BigSwap( HYPRE_BigInt *v, HYPRE_Int  i, HYPRE_Int  j );
void hypre_BigQsort0( HYPRE_BigInt *v, HYPRE_Int  left, HYPRE_Int  right );
void hypre_topo_sort(const HYPRE_Int *row_ptr, const HYPRE_Int *col_inds, const HYPRE_Complex *data, HYPRE_Int *ordering, HYPRE_Int n);
void hypre_dense_topo_sort(const HYPRE_Complex *L, HYPRE_Int *ordering, HYPRE_Int n, HYPRE_Int is_col_major);

/* qsplit.c */
HYPRE_Int hypre_DoubleQuickSplit ( HYPRE_Real *values , HYPRE_Int *indices , HYPRE_Int list_length , HYPRE_Int NumberKept );

/* random.c */
/* HYPRE_CUDA_GLOBAL */ void hypre_SeedRand ( HYPRE_Int seed );
/* HYPRE_CUDA_GLOBAL */ HYPRE_Int hypre_RandI ( void );
/* HYPRE_CUDA_GLOBAL */ HYPRE_Real hypre_Rand ( void );

/* hypre_prefix_sum.c */
/**
 * Assumed to be called within an omp region.
 * Let x_i be the input of ith thread.
 * The output of ith thread y_i = x_0 + x_1 + ... + x_{i-1}
 * Additionally, sum = x_0 + x_1 + ... + x_{nthreads - 1}
 * Note that always y_0 = 0
 *
 * @param workspace at least with length (nthreads+1)
 *                  workspace[tid] will contain result for tid
 *                  workspace[nthreads] will contain sum
 */
void hypre_prefix_sum(HYPRE_Int *in_out, HYPRE_Int *sum, HYPRE_Int *workspace);
/**
 * This version does prefix sum in pair.
 * Useful when we prefix sum of diag and offd in tandem.
 *
 * @param worksapce at least with length 2*(nthreads+1)
 *                  workspace[2*tid] and workspace[2*tid+1] will contain results for tid
 *                  workspace[3*nthreads] and workspace[3*nthreads + 1] will contain sums
 */
void hypre_prefix_sum_pair(HYPRE_Int *in_out1, HYPRE_Int *sum1, HYPRE_Int *in_out2, HYPRE_Int *sum2, HYPRE_Int *workspace);
/**
 * @param workspace at least with length 3*(nthreads+1)
 *                  workspace[3*tid:3*tid+3) will contain results for tid
 */
void hypre_prefix_sum_triple(HYPRE_Int *in_out1, HYPRE_Int *sum1, HYPRE_Int *in_out2, HYPRE_Int *sum2, HYPRE_Int *in_out3, HYPRE_Int *sum3, HYPRE_Int *workspace);

/**
 * n prefix-sums together.
 * workspace[n*tid:n*(tid+1)) will contain results for tid
 * workspace[nthreads*tid:nthreads*(tid+1)) will contain sums
 *
 * @param workspace at least with length n*(nthreads+1)
 */
void hypre_prefix_sum_multiple(HYPRE_Int *in_out, HYPRE_Int *sum, HYPRE_Int n, HYPRE_Int *workspace);

/* hypre_merge_sort.c */
/**
 * Why merge sort?
 * 1) Merge sort can take advantage of eliminating duplicates.
 * 2) Merge sort is more efficiently parallelizable than qsort
 */

/**
 * Out of place merge sort with duplicate elimination
 * @ret number of unique elements
 */
HYPRE_Int hypre_merge_sort_unique(HYPRE_Int *in, HYPRE_Int *out, HYPRE_Int len);
/**
 * Out of place merge sort with duplicate elimination
 *
 * @param out pointer to output can be in or temp
 * @ret number of unique elements
 */
HYPRE_Int hypre_merge_sort_unique2(HYPRE_Int *in, HYPRE_Int *temp, HYPRE_Int len, HYPRE_Int **out);

void hypre_merge_sort(HYPRE_Int *in, HYPRE_Int *temp, HYPRE_Int len, HYPRE_Int **sorted);

void hypre_union2(HYPRE_Int n1, HYPRE_BigInt *arr1, HYPRE_Int n2, HYPRE_BigInt *arr2, HYPRE_Int *n3, HYPRE_BigInt *arr3, HYPRE_Int *map1, HYPRE_Int *map2);

/* hypre_hopscotch_hash.c */

#ifdef HYPRE_USING_OPENMP

/* Check if atomic operations are available to use concurrent hopscotch hash table */
#if defined(__GNUC__) && defined(__GNUC_MINOR__) && defined(__GNUC_PATCHLEVEL__) && (__GNUC__ * 10000 + __GNUC_MINOR__ * 100 + __GNUC_PATCHLEVEL__) > 40100
#define HYPRE_USING_ATOMIC
//#elif defined _MSC_VER // JSP: haven't tested, so comment out for now
//#define HYPRE_USING_ATOMIC
//#elif defined(__STDC_VERSION__) && __STDC_VERSION__ >= 201112L && !defined(__STDC_NO_ATOMICS__)
// JSP: not many compilers have implemented this, so comment out for now
//#define HYPRE_USING_ATOMIC
//#include <stdatomic.h>
#endif

#endif // HYPRE_USING_OPENMP

#ifdef HYPRE_HOPSCOTCH
#ifdef HYPRE_USING_ATOMIC
// concurrent hopscotch hashing is possible only with atomic supports
#define HYPRE_CONCURRENT_HOPSCOTCH
#endif
#endif

#ifdef HYPRE_CONCURRENT_HOPSCOTCH
typedef struct {
  HYPRE_Int volatile timestamp;
  omp_lock_t         lock;
} hypre_HopscotchSegment;
#endif

/**
 * The current typical use case of unordered set is putting input sequence
 * with lots of duplication (putting all colidx received from other ranks),
 * followed by one sweep of enumeration.
 * Since the capacity is set to the number of inputs, which is much larger
 * than the number of unique elements, we optimize for initialization and
 * enumeration whose time is proportional to the capacity.
 * For initialization and enumeration, structure of array (SoA) is better
 * for vectorization, cache line utilization, and so on.
 */
typedef struct
{
   HYPRE_Int  volatile              segmentMask;
   HYPRE_Int  volatile              bucketMask;
#ifdef HYPRE_CONCURRENT_HOPSCOTCH
   hypre_HopscotchSegment* volatile segments;
#endif
   HYPRE_Int *volatile              key;
   hypre_uint *volatile             hopInfo;
   HYPRE_Int *volatile              hash;
} hypre_UnorderedIntSet;

typedef struct
{
   HYPRE_Int volatile            segmentMask;
   HYPRE_Int volatile            bucketMask;
#ifdef HYPRE_CONCURRENT_HOPSCOTCH
   hypre_HopscotchSegment* volatile segments;
#endif
   HYPRE_BigInt *volatile           key;
   hypre_uint *volatile             hopInfo;
   HYPRE_BigInt *volatile           hash;
} hypre_UnorderedBigIntSet;

typedef struct
{
  hypre_uint volatile hopInfo;
  HYPRE_Int  volatile hash;
  HYPRE_Int  volatile key;
  HYPRE_Int  volatile data;
} hypre_HopscotchBucket;

typedef struct
{
  hypre_uint volatile hopInfo;
  HYPRE_BigInt  volatile hash;
  HYPRE_BigInt  volatile key;
  HYPRE_Int  volatile data;
} hypre_BigHopscotchBucket;

/**
 * The current typical use case of unoredered map is putting input sequence
 * with no duplication (inverse map of a bijective mapping) followed by
 * lots of lookups.
 * For lookup, array of structure (AoS) gives better cache line utilization.
 */
typedef struct
{
   HYPRE_Int  volatile              segmentMask;
   HYPRE_Int  volatile              bucketMask;
#ifdef HYPRE_CONCURRENT_HOPSCOTCH
   hypre_HopscotchSegment* volatile segments;
#endif
   hypre_HopscotchBucket* volatile table;
} hypre_UnorderedIntMap;

typedef struct
{
   HYPRE_Int  volatile                segmentMask;
   HYPRE_Int  volatile                bucketMask;
#ifdef HYPRE_CONCURRENT_HOPSCOTCH
   hypre_HopscotchSegment*   volatile segments;
#endif
   hypre_BigHopscotchBucket* volatile table;
} hypre_UnorderedBigIntMap;

/**
 * Sort array "in" with length len and put result in array "out"
 * "in" will be deallocated unless in == *out
 * inverse_map is an inverse hash table s.t. inverse_map[i] = j iff (*out)[j] = i
 */
void hypre_sort_and_create_inverse_map(
  HYPRE_Int *in, HYPRE_Int len, HYPRE_Int **out, hypre_UnorderedIntMap *inverse_map);

#ifdef HYPRE_CONCURRENT_HOPSCOTCH
void hypre_big_merge_sort(HYPRE_BigInt *in, HYPRE_BigInt *temp, HYPRE_Int len, HYPRE_BigInt **sorted);
void hypre_big_sort_and_create_inverse_map(HYPRE_BigInt *in, HYPRE_Int len, HYPRE_BigInt **out, hypre_UnorderedBigIntMap *inverse_map);
#endif


/* hypre_cuda_utils.c */
#if defined(HYPRE_USING_CUDA)
#ifdef __cplusplus
extern "C++" {
#endif
dim3 hypre_GetDefaultCUDABlockDimension();

dim3 hypre_GetDefaultCUDAGridDimension( HYPRE_Int n, const char *granularity, dim3 bDim );

template <typename T1, typename T2, typename T3> HYPRE_Int hypreDevice_StableSortByTupleKey(HYPRE_Int N, T1 *keys1, T2 *keys2, T3 *vals, HYPRE_Int opt);

#ifdef __cplusplus
}
#endif

HYPRE_Int hypreDevice_GetRowNnz(HYPRE_Int nrows, HYPRE_Int *d_row_indices, HYPRE_Int *d_diag_ia, HYPRE_Int *d_offd_ia, HYPRE_Int *d_rownnz);

HYPRE_Int hypreDevice_CopyParCSRRows(HYPRE_Int nrows, HYPRE_Int *d_row_indices, HYPRE_Int job, HYPRE_Int has_offd, HYPRE_Int first_col, HYPRE_Int *d_col_map_offd_A, HYPRE_Int *d_diag_i, HYPRE_Int *d_diag_j, HYPRE_Complex *d_diag_a, HYPRE_Int *d_offd_i, HYPRE_Int *d_offd_j, HYPRE_Complex *d_offd_a, HYPRE_Int *d_ib, HYPRE_BigInt *d_jb, HYPRE_Complex *d_ab);

HYPRE_Int hypreDevice_IntegerReduceSum(HYPRE_Int m, HYPRE_Int *d_i);

HYPRE_Int hypreDevice_IntegerInclusiveScan(HYPRE_Int n, HYPRE_Int *d_i);

HYPRE_Int hypreDevice_IntegerExclusiveScan(HYPRE_Int n, HYPRE_Int *d_i);

HYPRE_Int* hypreDevice_CsrRowPtrsToIndices(HYPRE_Int nrows, HYPRE_Int nnz, HYPRE_Int *d_row_ptr);

HYPRE_Int hypreDevice_CsrRowPtrsToIndices_v2(HYPRE_Int nrows, HYPRE_Int *d_row_ptr, HYPRE_Int *d_row_ind);

HYPRE_Int hypreDevice_CsrRowPtrsToIndicesWithRowNum(HYPRE_Int nrows, HYPRE_Int *d_row_ptr, HYPRE_Int *d_row_num, HYPRE_Int *d_row_ind);

HYPRE_Int* hypreDevice_CsrRowIndicesToPtrs(HYPRE_Int nrows, HYPRE_Int nnz, HYPRE_Int *d_row_ind);

HYPRE_Int hypreDevice_CsrRowIndicesToPtrs_v2(HYPRE_Int nrows, HYPRE_Int nnz, HYPRE_Int *d_row_ind, HYPRE_Int *d_row_ptr);

HYPRE_Int hypreDevice_GenScatterAdd(HYPRE_Real *x, HYPRE_Int ny, HYPRE_Int *map, HYPRE_Real *y);

HYPRE_Int hypreDevice_ScatterConstant(HYPRE_Int *x, HYPRE_Int n, HYPRE_Int *map, HYPRE_Int v);

HYPRE_Int hypreDevice_IVAXPY(HYPRE_Int n, HYPRE_Complex *a, HYPRE_Complex *x, HYPRE_Complex *y);

HYPRE_Int hypreDevice_DiagScaleVector(HYPRE_Int n, HYPRE_Int *A_i, HYPRE_Complex *A_data, HYPRE_Complex *x, HYPRE_Complex *y);

HYPRE_Int hypreDevice_BigToSmallCopy(HYPRE_Int *tgt, const HYPRE_BigInt *src, HYPRE_Int size);

void hypre_CudaCompileFlagCheck();

#endif


#ifdef __cplusplus
}
#endif

#endif

