# Interface definition for thorn Accelerator

IMPLEMENTS: Accelerator

USES INCLUDE HEADER: carpet.hh



# Adaptive mesh refinement and multi-block

CCTK_INT FUNCTION GetRefinementLevel            \
  (CCTK_POINTER_TO_CONST IN cctkGH)
USES FUNCTION GetRefinementLevel
CCTK_INT FUNCTION GetRefinementLevels           \
  (CCTK_POINTER_TO_CONST IN cctkGH)
USES FUNCTION GetRefinementLevels
CCTK_INT FUNCTION GetLocalComponents            \
  (CCTK_POINTER_TO_CONST IN cctkGH)
USES FUNCTION GetLocalComponents

CCTK_INT FUNCTION MultiPatch_GetSystemSpecification \
  (CCTK_INT OUT maps)
USES FUNCTION MultiPatch_GetSystemSpecification



# Functions for hooks provided by Carpet and MoL

CCTK_INT FUNCTION Accelerator_Cycle             \
  (CCTK_POINTER_TO_CONST IN cctkGH)
PROVIDES FUNCTION Accelerator_Cycle             \
  WITH AcceleratorThorn_Cycle                   \
  LANGUAGE C
  
CCTK_INT FUNCTION Accelerator_CopyFromPast      \
  (CCTK_POINTER_TO_CONST IN cctkGH,             \
   CCTK_INT ARRAY        IN vars,               \
   CCTK_INT              IN nvars)
PROVIDES FUNCTION Accelerator_CopyFromPast      \
  WITH AcceleratorThorn_CopyFromPast            \
  LANGUAGE C

CCTK_INT FUNCTION Accelerator_PreSync           \
  (CCTK_POINTER_TO_CONST IN cctkGH,             \
   CCTK_INT ARRAY        IN groups,             \
   CCTK_INT              IN ngroups)
PROVIDES FUNCTION Accelerator_PreSync           \
  WITH AcceleratorThorn_PreSync                 \
  LANGUAGE C

CCTK_INT FUNCTION Accelerator_PostSync          \
  (CCTK_POINTER_TO_CONST IN cctkGH,             \
   CCTK_INT ARRAY        IN groups,             \
   CCTK_INT              IN ngroups)
PROVIDES FUNCTION Accelerator_PostSync          \
  WITH AcceleratorThorn_PostSync                \
  LANGUAGE C

CCTK_INT FUNCTION Accelerator_PreCallFunction   \
  (CCTK_POINTER_TO_CONST IN cctkGH,             \
   CCTK_POINTER_TO_CONST IN attribute)
PROVIDES FUNCTION Accelerator_PreCallFunction   \
  WITH AcceleratorThorn_PreCallFunction         \
  LANGUAGE C

CCTK_INT FUNCTION Accelerator_PostCallFunction  \
  (CCTK_POINTER_TO_CONST IN cctkGH,             \
   CCTK_POINTER_TO_CONST IN attribute)
PROVIDES FUNCTION Accelerator_PostCallFunction  \
  WITH AcceleratorThorn_PostCallFunction        \
  LANGUAGE C

CCTK_INT FUNCTION Accelerator_NotifyDataModified        \
  (CCTK_POINTER_TO_CONST IN cctkGH,                     \
   CCTK_INT ARRAY        IN variables,                  \
   CCTK_INT ARRAY        IN rls,                        \
   CCTK_INT ARRAY        IN tls,                        \
   CCTK_INT              IN nvariables,                 \
   CCTK_INT              IN on_device)
PROVIDES FUNCTION Accelerator_NotifyDataModified        \
  WITH AcceleratorThorn_NotifyDataModified              \
  LANGUAGE C

CCTK_INT FUNCTION Accelerator_RequireInvalidData        \
  (CCTK_POINTER_TO_CONST IN cctkGH,                     \
   CCTK_INT ARRAY        IN variables,                  \
   CCTK_INT ARRAY        IN rls,                        \
   CCTK_INT ARRAY        IN tls,                        \
   CCTK_INT              IN nvariables,                 \
   CCTK_INT              IN on_device)
PROVIDES FUNCTION Accelerator_RequireInvalidData        \
  WITH AcceleratorThorn_RequireInvalidData              \
  LANGUAGE C

CCTK_INT FUNCTION Accelerator_RequireValidData  \
  (CCTK_POINTER_TO_CONST IN cctkGH,             \
   CCTK_INT ARRAY        IN variables,          \
   CCTK_INT ARRAY        IN rls,                \
   CCTK_INT ARRAY        IN tls,                \
   CCTK_INT              IN nvariables,         \
   CCTK_INT              IN on_device)
PROVIDES FUNCTION Accelerator_RequireValidData  \
  WITH AcceleratorThorn_RequireValidData        \
  LANGUAGE C



# Low-level driver functions, to be implemented by CUDA/OpenCL drivers

# 
CCTK_INT FUNCTION Device_GetDevice              \
  (CCTK_POINTER_TO_CONST IN cctkGH)
USES FUNCTION Device_GetDevice

# Create (i.e. begin to track) a set of variables/timelevels
# (This routine will be called exactly once before a variable/timelevel
#  combination is mentioned to the device.)
CCTK_INT FUNCTION Device_CreateVariables        \
  (CCTK_POINTER_TO_CONST IN cctkGH,             \
   CCTK_INT ARRAY IN vars,                      \
   CCTK_INT ARRAY IN rls,                       \
   CCTK_INT ARRAY IN tls,                       \
   CCTK_INT IN nvars)
REQUIRES FUNCTION Device_CreateVariables

# Cycle all timelevels on the device
# (This routine should not block.)
CCTK_INT FUNCTION Device_CopyCycle              \
  (CCTK_POINTER_TO_CONST IN cctkGH,             \
   CCTK_INT ARRAY IN vars,                      \
   CCTK_INT ARRAY IN rls,                       \
   CCTK_INT ARRAY IN tls,                       \
   CCTK_INT IN nvars)
REQUIRES FUNCTION Device_CopyCycle

# Copy past to current time level on the device
# (This routine should not block.)
CCTK_INT FUNCTION Device_CopyFromPast           \
  (CCTK_POINTER_TO_CONST IN cctkGH,             \
   CCTK_INT ARRAY IN vars,                      \
   CCTK_INT ARRAY IN rls,                       \
   CCTK_INT ARRAY IN tls,                       \
   CCTK_INT IN nvars)
REQUIRES FUNCTION Device_CopyFromPast

# Copy data from the host to the device.
# The return argument indicates whether the data have been copied or moved.
# (This routine should not block.)
CCTK_INT FUNCTION Device_CopyToDevice           \
  (CCTK_POINTER_TO_CONST IN cctkGH,             \
   CCTK_INT ARRAY IN vars,                      \
   CCTK_INT ARRAY IN rls,                       \
   CCTK_INT ARRAY IN tls,                       \
   CCTK_INT IN nvars,                           \
   CCTK_INT OUT moved)
REQUIRES FUNCTION Device_CopyToDevice

# Copy data from the device back to the host.
# The return argument indicates whether the data have been copied or moved.
# (This routine should block until all data have been copied.)
CCTK_INT FUNCTION Device_CopyToHost             \
  (CCTK_POINTER_TO_CONST IN cctkGH,             \
   CCTK_INT ARRAY IN vars,                      \
   CCTK_INT ARRAY IN rls,                       \
   CCTK_INT ARRAY IN tls,                       \
   CCTK_INT IN nvars,                           \
   CCTK_INT OUT moved)
REQUIRES FUNCTION Device_CopyToHost

# Copy those data from the device back to the host that will be
# needed for synchronization (i.e. inter-process synchronization;
# AMR is not yet supported).
# (This routine should block until all data have been copied.)
CCTK_INT FUNCTION Device_CopyPreSync            \
  (CCTK_POINTER_TO_CONST IN cctkGH,             \
   CCTK_INT ARRAY IN vars,                      \
   CCTK_INT ARRAY IN rls,                       \
   CCTK_INT ARRAY IN tls,                       \
   CCTK_INT IN nvars)
REQUIRES FUNCTION Device_CopyPreSync

# Copy the ghost zones to the device.
# (This routine should not block.)
CCTK_INT FUNCTION Device_CopyPostSync           \
  (CCTK_POINTER_TO_CONST IN cctkGH,             \
   CCTK_INT ARRAY IN vars,                      \
   CCTK_INT ARRAY IN rls,                       \
   CCTK_INT ARRAY IN tls,                       \
   CCTK_INT IN nvars)
REQUIRES FUNCTION Device_CopyPostSync