CudaUtils.C File Reference

#include <stdio.h>
#include "common.h"
#include "charm++.h"
#include "CudaUtils.h"

Go to the source code of this file.

Functions
void	cudaDie (const char *msg, cudaError_t err)
void	curandDie (const char *msg, int err)
void	cudaNAMD_bug (const char *msg)
void	cuda_affinity_initialize ()
void	clear_device_array_async_T (void *data, const size_t ndata, cudaStream_t stream, const size_t sizeofT)
void	clear_device_array_T (void *data, const size_t ndata, const size_t sizeofT)
void	allocate_host_T (void **pp, const size_t len, const size_t sizeofT)
void	allocate_device_T (void **pp, const size_t len, const size_t sizeofT)
void	allocate_device_T_managed (void **pp, const size_t len, const size_t sizeofT)
void	allocate_device_T_async (void **pp, const size_t len, const size_t sizeofT, cudaStream_t stream)
void	deallocate_device_T (void **pp)
void	deallocate_device_T_async (void **pp, cudaStream_t stream)
void	deallocate_host_T (void **pp)
bool	reallocate_device_T (void **pp, size_t *curlen, const size_t newlen, const float fac, const size_t sizeofT)
bool	reallocate_host_T (void **pp, size_t *curlen, const size_t newlen, const float fac, const unsigned int flag, const size_t sizeofT)
void	copy_HtoD_async_T (const void *h_array, void *d_array, size_t array_len, cudaStream_t stream, const size_t sizeofT)
void	copy_HtoD_T (const void *h_array, void *d_array, size_t array_len, const size_t sizeofT)
void	copy_DtoH_async_T (const void *d_array, void *h_array, const size_t array_len, cudaStream_t stream, const size_t sizeofT)
void	copy_DtoH_T (const void *d_array, void *h_array, const size_t array_len, const size_t sizeofT)
void	copy_DtoD_async_T (const void *d_src, void *d_dst, const size_t array_len, cudaStream_t stream, const size_t sizeofT)
void	copy_DtoD_T (const void *d_src, void *d_dst, const size_t array_len, const size_t sizeofT)
void	copy_PeerDtoD_async_T (const int src_dev, const int dst_dev, const void *d_src, void *d_dst, const size_t array_len, cudaStream_t stream, const size_t sizeofT)
void	copy3D_HtoD_T (void *src_data, void *dst_data, int src_x0, int src_y0, int src_z0, size_t src_xsize, size_t src_ysize, int dst_x0, int dst_y0, int dst_z0, size_t dst_xsize, size_t dst_ysize, size_t width, size_t height, size_t depth, size_t sizeofT, cudaStream_t stream)
void	copy3D_DtoH_T (void *src_data, void *dst_data, int src_x0, int src_y0, int src_z0, size_t src_xsize, size_t src_ysize, int dst_x0, int dst_y0, int dst_z0, size_t dst_xsize, size_t dst_ysize, size_t width, size_t height, size_t depth, size_t sizeofT, cudaStream_t stream)
void	copy3D_DtoD_T (void *src_data, void *dst_data, int src_x0, int src_y0, int src_z0, size_t src_xsize, size_t src_ysize, int dst_x0, int dst_y0, int dst_z0, size_t dst_xsize, size_t dst_ysize, size_t width, size_t height, size_t depth, size_t sizeofT, cudaStream_t stream)
void	copy3D_PeerDtoD_T (int src_dev, int dst_dev, void *src_data, void *dst_data, int src_x0, int src_y0, int src_z0, size_t src_xsize, size_t src_ysize, int dst_x0, int dst_y0, int dst_z0, size_t dst_xsize, size_t dst_ysize, size_t width, size_t height, size_t depth, size_t sizeofT, cudaStream_t stream)

Function Documentation

allocate_device_T()

void allocate_device_T	(	void **	pp,
		const size_t	len,
		const size_t	sizeofT
	)

Definition at line 97 of file CudaUtils.C.

References cudaCheck.

Referenced by allocate_device(), allocate_device_T_async(), and bindTextureObject().

                                                                          {
  cudaCheck(cudaMalloc(pp, sizeofT*len));
}

allocate_device_T_async()

void allocate_device_T_async	(	void **	pp,
		const size_t	len,
		const size_t	sizeofT,
		cudaStream_t	stream
	)

Definition at line 105 of file CudaUtils.C.

References allocate_device_T(), and cudaCheck.

Referenced by allocate_device_async().

                                                                                                    {
#if (CUDART_VERSION >= 11020)
  cudaCheck(cudaMallocAsync(pp, sizeofT*len, stream));
#else
  allocate_device_T(pp, len, sizeofT);
#endif
}

allocate_device_T_managed()

void allocate_device_T_managed	(	void **	pp,
		const size_t	len,
		const size_t	sizeofT
	)

Definition at line 101 of file CudaUtils.C.

References cudaCheck.

Referenced by allocate_device_managed().

                                                                                 {
  cudaCheck(cudaMallocManaged(pp, sizeofT*len));
}

allocate_host_T()

void allocate_host_T	(	void **	pp,
		const size_t	len,
		const size_t	sizeofT
	)

Definition at line 87 of file CudaUtils.C.

References cudaCheck.

Referenced by allocate_host().

                                                                        {
  cudaCheck(cudaMallocHost(pp, sizeofT*len));
}

clear_device_array_async_T()

void clear_device_array_async_T	(	void *	data,
		const size_t	ndata,
		cudaStream_t	stream,
		const size_t	sizeofT
	)

Definition at line 73 of file CudaUtils.C.

References cudaCheck.

Referenced by clear_device_array().

                                                                                                           {
  cudaCheck(cudaMemsetAsync(data, 0, sizeofT*ndata, stream));
}

clear_device_array_T()

void clear_device_array_T	(	void *	data,
		const size_t	ndata,
		const size_t	sizeofT
	)

Definition at line 77 of file CudaUtils.C.

References cudaCheck.

Referenced by clear_device_array_sync().

                                                                                {
  cudaCheck(cudaMemset(data, 0, sizeofT*ndata));
}

copy3D_DtoD_T()

void copy3D_DtoD_T	(	void *	src_data,
		void *	dst_data,
		int	src_x0,
		int	src_y0,
		int	src_z0,
		size_t	src_xsize,
		size_t	src_ysize,
		int	dst_x0,
		int	dst_y0,
		int	dst_z0,
		size_t	dst_xsize,
		size_t	dst_ysize,
		size_t	width,
		size_t	height,
		size_t	depth,
		size_t	sizeofT,
		cudaStream_t	stream
	)

Definition at line 319 of file CudaUtils.C.

References cudaCheck.

Referenced by copy3D_DtoD().

                                       {
  cudaMemcpy3DParms parms = {0};

  parms.srcPos = make_cudaPos(sizeofT*src_x0, src_y0, src_z0);
  parms.srcPtr = make_cudaPitchedPtr(src_data, sizeofT*src_xsize, src_xsize, src_ysize);

  parms.dstPos = make_cudaPos(sizeofT*dst_x0, dst_y0, dst_z0);
  parms.dstPtr = make_cudaPitchedPtr(dst_data, sizeofT*dst_xsize, dst_xsize, dst_ysize);

  parms.extent = make_cudaExtent(sizeofT*width, height, depth);
  parms.kind = cudaMemcpyDeviceToDevice;

  cudaCheck(cudaMemcpy3DAsync(&parms, stream));
}

copy3D_DtoH_T()

void copy3D_DtoH_T	(	void *	src_data,
		void *	dst_data,
		int	src_x0,
		int	src_y0,
		int	src_z0,
		size_t	src_xsize,
		size_t	src_ysize,
		int	dst_x0,
		int	dst_y0,
		int	dst_z0,
		size_t	dst_xsize,
		size_t	dst_ysize,
		size_t	width,
		size_t	height,
		size_t	depth,
		size_t	sizeofT,
		cudaStream_t	stream
	)

Definition at line 294 of file CudaUtils.C.

References cudaCheck.

Referenced by copy3D_DtoH().

                                       {
  cudaMemcpy3DParms parms = {0};

  parms.srcPos = make_cudaPos(sizeofT*src_x0, src_y0, src_z0);
  parms.srcPtr = make_cudaPitchedPtr(src_data, sizeofT*src_xsize, src_xsize, src_ysize);

  parms.dstPos = make_cudaPos(sizeofT*dst_x0, dst_y0, dst_z0);
  parms.dstPtr = make_cudaPitchedPtr(dst_data, sizeofT*dst_xsize, dst_xsize, dst_ysize);

  parms.extent = make_cudaExtent(sizeofT*width, height, depth);
  parms.kind = cudaMemcpyDeviceToHost;

  cudaCheck(cudaMemcpy3DAsync(&parms, stream));
}

copy3D_HtoD_T()

void copy3D_HtoD_T	(	void *	src_data,
		void *	dst_data,
		int	src_x0,
		int	src_y0,
		int	src_z0,
		size_t	src_xsize,
		size_t	src_ysize,
		int	dst_x0,
		int	dst_y0,
		int	dst_z0,
		size_t	dst_xsize,
		size_t	dst_ysize,
		size_t	width,
		size_t	height,
		size_t	depth,
		size_t	sizeofT,
		cudaStream_t	stream
	)

Definition at line 269 of file CudaUtils.C.

References cudaCheck.

Referenced by copy3D_HtoD().

                                       {
  cudaMemcpy3DParms parms = {0};

  parms.srcPos = make_cudaPos(sizeofT*src_x0, src_y0, src_z0);
  parms.srcPtr = make_cudaPitchedPtr(src_data, sizeofT*src_xsize, src_xsize, src_ysize);

  parms.dstPos = make_cudaPos(sizeofT*dst_x0, dst_y0, dst_z0);
  parms.dstPtr = make_cudaPitchedPtr(dst_data, sizeofT*dst_xsize, dst_xsize, dst_ysize);

  parms.extent = make_cudaExtent(sizeofT*width, height, depth);
  parms.kind = cudaMemcpyHostToDevice;

  cudaCheck(cudaMemcpy3DAsync(&parms, stream));
}

copy3D_PeerDtoD_T()

void copy3D_PeerDtoD_T	(	int	src_dev,
		int	dst_dev,
		void *	src_data,
		void *	dst_data,
		int	src_x0,
		int	src_y0,
		int	src_z0,
		size_t	src_xsize,
		size_t	src_ysize,
		int	dst_x0,
		int	dst_y0,
		int	dst_z0,
		size_t	dst_xsize,
		size_t	dst_ysize,
		size_t	width,
		size_t	height,
		size_t	depth,
		size_t	sizeofT,
		cudaStream_t	stream
	)

Definition at line 344 of file CudaUtils.C.

References cudaCheck, and cudaDie().

Referenced by copy3D_PeerDtoD().

                                       {
#ifdef NAMD_HIP
// TODO-HIP: Is a workaround possible? cudaMemcpy3D+cudaMemcpyPeer+cudaMemcpy3D
   cudaDie("cudaMemcpy3DPeerAsync is not supported by HIP");
#else
  cudaMemcpy3DPeerParms parms = {0};

  parms.srcDevice = src_dev;
  parms.dstDevice = dst_dev;

  parms.srcPos = make_cudaPos(sizeofT*src_x0, src_y0, src_z0);
  parms.srcPtr = make_cudaPitchedPtr(src_data, sizeofT*src_xsize, src_xsize, src_ysize);

  parms.dstPos = make_cudaPos(sizeofT*dst_x0, dst_y0, dst_z0);
  parms.dstPtr = make_cudaPitchedPtr(dst_data, sizeofT*dst_xsize, dst_xsize, dst_ysize);

  parms.extent = make_cudaExtent(sizeofT*width, height, depth);

  cudaCheck(cudaMemcpy3DPeerAsync(&parms, stream));
#endif
}

copy_DtoD_async_T()

void copy_DtoD_async_T	(	const void *	d_src,
		void *	d_dst,
		const size_t	array_len,
		cudaStream_t	stream,
		const size_t	sizeofT
	)

Definition at line 246 of file CudaUtils.C.

References cudaCheck.

Referenced by copy_DtoD().

                                 {
  cudaCheck(cudaMemcpyAsync(d_dst, d_src, sizeofT*array_len, cudaMemcpyDeviceToDevice, stream));
}

copy_DtoD_T()

void copy_DtoD_T	(	const void *	d_src,
		void *	d_dst,
		const size_t	array_len,
		const size_t	sizeofT
	)

Definition at line 251 of file CudaUtils.C.

References cudaCheck.

Referenced by copy_DtoD_sync().

                                                                                               {
  cudaCheck(cudaMemcpy(d_dst, d_src, sizeofT*array_len, cudaMemcpyDeviceToDevice));
}

copy_DtoH_async_T()

void copy_DtoH_async_T	(	const void *	d_array,
		void *	h_array,
		const size_t	array_len,
		cudaStream_t	stream,
		const size_t	sizeofT
	)

Definition at line 233 of file CudaUtils.C.

References cudaCheck.

Referenced by copy_DtoH().

                                 {
  cudaCheck(cudaMemcpyAsync(h_array, d_array, sizeofT*array_len, cudaMemcpyDeviceToHost, stream));
}

copy_DtoH_T()

void copy_DtoH_T	(	const void *	d_array,
		void *	h_array,
		const size_t	array_len,
		const size_t	sizeofT
	)

Definition at line 238 of file CudaUtils.C.

References cudaCheck.

Referenced by copy_DtoH_sync().

                                                                                                   {
  cudaCheck(cudaMemcpy(h_array, d_array, sizeofT*array_len, cudaMemcpyDeviceToHost));
}

copy_HtoD_async_T()

void copy_HtoD_async_T	(	const void *	h_array,
		void *	d_array,
		size_t	array_len,
		cudaStream_t	stream,
		const size_t	sizeofT
	)

Definition at line 219 of file CudaUtils.C.

References cudaCheck.

Referenced by copy_HtoD().

                                 {
  cudaCheck(cudaMemcpyAsync(d_array, h_array, sizeofT*array_len, cudaMemcpyHostToDevice, stream));
}

copy_HtoD_T()

void copy_HtoD_T	(	const void *	h_array,
		void *	d_array,
		size_t	array_len,
		const size_t	sizeofT
	)

Definition at line 224 of file CudaUtils.C.

References cudaCheck.

Referenced by bindTextureObject(), and copy_HtoD_sync().

                           {
  cudaCheck(cudaMemcpy(d_array, h_array, sizeofT*array_len, cudaMemcpyHostToDevice));
}

copy_PeerDtoD_async_T()

void copy_PeerDtoD_async_T	(	const int	src_dev,
		const int	dst_dev,
		const void *	d_src,
		void *	d_dst,
		const size_t	array_len,
		cudaStream_t	stream,
		const size_t	sizeofT
	)

Definition at line 259 of file CudaUtils.C.

References cudaCheck.

Referenced by copy_PeerDtoD().

                        {
  cudaCheck(cudaMemcpyPeerAsync(d_dst, dst_dev, d_src, src_dev, sizeofT*array_len, stream));
}

cuda_affinity_initialize()

void cuda_affinity_initialize

(

)

Definition at line 55 of file CudaUtils.C.

Referenced by all_init().

                                {
  int devcnt = 0;
  cudaError_t err = cudaGetDeviceCount(&devcnt);
  if ( devcnt == 1 ) {  // only one device so it must be ours
    int *dummy;
    if ( err == cudaSuccess ) err = cudaSetDevice(0);
    if ( err == cudaSuccess ) err = cudaSetDeviceFlags(cudaDeviceMapHost);
    if ( err == cudaSuccess ) err = cudaMalloc(&dummy, 4);
  }
  if ( err != cudaSuccess ) {
    char host[128];
    gethostname(host, 128);  host[127] = 0;
    fprintf(stderr,"CUDA initialization error on %s: %s\n", host, cudaGetErrorString(err));
  }
}

cudaDie()

void cudaDie	(	const char *	msg,
		cudaError_t	err
	)

Definition at line 9 of file CudaUtils.C.

References NAMD_die().

Referenced by copy3D_PeerDtoD_T(), cuda_check_pme_charges(), cuda_check_pme_forces(), and DeviceCUDA::initialize().

                                               {
  char host[128];
  gethostname(host, 128);  host[127] = 0;
  char devstr[128] = "";
  int devnum;
  if ( cudaGetDevice(&devnum) == cudaSuccess ) {
    sprintf(devstr, " device %d", devnum);
  }
  cudaDeviceProp deviceProp;
  if ( cudaGetDeviceProperties(&deviceProp, devnum) == cudaSuccess ) {
    sprintf(devstr, " device %d pci %x:%x:%x", devnum,
      deviceProp.pciDomainID, deviceProp.pciBusID, deviceProp.pciDeviceID);
  }
  char errmsg[1024];
  if (err == cudaSuccess) {
    sprintf(errmsg,"CUDA error %s on Pe %d (%s%s)", msg, CkMyPe(), host, devstr);
  } else {
    sprintf(errmsg,"CUDA error %s on Pe %d (%s%s): %s", msg, CkMyPe(), host, devstr, cudaGetErrorString(err));    
  }
  NAMD_die(errmsg);
}

cudaNAMD_bug()

void cudaNAMD_bug

(

const char *

msg

)

Definition at line 53 of file CudaUtils.C.

References NAMD_bug().

Referenced by CudaFFTCompute::backward(), and CudaFFTCompute::forward().

{NAMD_bug(msg);}

curandDie()

void curandDie	(	const char *	msg,
		int	err
	)

Definition at line 31 of file CudaUtils.C.

References NAMD_die().

                                         {
  char host[128];
  gethostname(host, 128);  host[127] = 0;
  char devstr[128] = "";
  int devnum;
  if ( cudaGetDevice(&devnum) == cudaSuccess ) {
    sprintf(devstr, " device %d", devnum);
  }
  cudaDeviceProp deviceProp;
  if ( cudaGetDeviceProperties(&deviceProp, devnum) == cudaSuccess ) {
    sprintf(devstr, " device %d pci %x:%x:%x", devnum,
      deviceProp.pciDomainID, deviceProp.pciBusID, deviceProp.pciDeviceID);
  }
  char errmsg[1024];
  if (err == cudaSuccess) {
    sprintf(errmsg,"CUDA cuRAND error %s on Pe %d (%s%s)", msg, CkMyPe(), host, devstr);
  } else {
    sprintf(errmsg,"CUDA cuRAND error %s on Pe %d (%s%s): status value %d", msg, CkMyPe(), host, devstr, err);    
  }
  NAMD_die(errmsg);
}

deallocate_device_T()

void deallocate_device_T

(

void **

pp

)

Definition at line 118 of file CudaUtils.C.

References cudaCheck.

Referenced by deallocate_device(), and deallocate_device_T_async().

                                    {
  
  if (*pp != NULL) {
    cudaCheck(cudaFree((void *)(*pp)));
    *pp = NULL;
  }

}

deallocate_device_T_async()

void deallocate_device_T_async	(	void **	pp,
		cudaStream_t	stream
	)

Definition at line 127 of file CudaUtils.C.

References cudaCheck, and deallocate_device_T().

Referenced by deallocate_device_async().

                                                               {
#if (CUDART_VERSION >= 11020)
  if (*pp != NULL) {
    cudaCheck(cudaFreeAsync((void *)(*pp), stream));
    *pp = NULL;
  }
#else
  deallocate_device_T(pp);
#endif
}

deallocate_host_T()

void deallocate_host_T

(

void **

pp

)

Definition at line 142 of file CudaUtils.C.

References cudaCheck.

Referenced by deallocate_host().

                                  {
  
  if (*pp != NULL) {
    cudaCheck(cudaFreeHost((void *)(*pp)));
    *pp = NULL;
  }

}

reallocate_device_T()

bool reallocate_device_T	(	void **	pp,
		size_t *	curlen,
		const size_t	newlen,
		const float	fac,
		const size_t	sizeofT
	)

Definition at line 161 of file CudaUtils.C.

References cudaCheck.

Referenced by reallocate_device().

                                                                                                                {

  if (*pp != NULL && *curlen < newlen) {
    cudaCheck(cudaFree((void *)(*pp)));
    *pp = NULL;
  }

  if (*pp == NULL) {
    if (fac > 1.0f) {
      *curlen = (size_t)(((double)(newlen))*(double)fac);
    } else {
      *curlen = newlen;
    }
    cudaCheck(cudaMalloc(pp, sizeofT*(*curlen)));
    return true;
  }

  return false;
}

reallocate_host_T()

bool reallocate_host_T	(	void **	pp,
		size_t *	curlen,
		const size_t	newlen,
		const float	fac,
		const unsigned int	flag,
		const size_t	sizeofT
	)

Definition at line 194 of file CudaUtils.C.

References cudaCheck.

Referenced by reallocate_host().

                                                                                       {

  if (*pp != NULL && *curlen < newlen) {
    cudaCheck(cudaFreeHost((void *)(*pp)));
    *pp = NULL;
  }

  if (*pp == NULL) {
    if (fac > 1.0f) {
      *curlen = (size_t)(((double)(newlen))*(double)fac);
    } else {
      *curlen = newlen;
    }
    cudaCheck(cudaHostAlloc(pp, sizeofT*(*curlen), flag));
    return true;
  }

  return false;
}