\input{pygmentsheader}

\section{Thorn Structure}

\frame{ \frametitle{}
 \hspace{1em}
 \begin{centering}
 \hspace{-1cm}\includegraphics[height=3.3cm]{cactuslogo_big}\\
 \vspace{1cm}
 {\Huge Writing Cactus Thorns}\\
 \end{centering}
}

\frame{ \frametitle{}
 {\Large Plan:
 \begin{itemize}
 \item Thorn structure
 \item HelloWorld thorn
 \item Conway's Game of Life
 \begin{itemize}
   \item Standalone code
   \item The \tt{Life} thorn
 \end{itemize}
 \item Solving the Wave Equation
 \begin{itemize}
   \item Standalone code 
   \item The BadWave thorn
   \item Advanced thorns: PUGH, MoL, AMR
 \end{itemize}
 \end{itemize}}
}

\subsection{Thorn Structure}

\frame { \frametitle{Thorn Structure}
Inside view of a plug-in module, or thorn for Cactus
\begin{figure}
\vspace{0.0cm}
\hspace{0.0cm}
\centering
\includegraphics[height=0.55\textwidth]{thorn_arch}
\end{figure}
}

\frame { \frametitle{Thorn Specification}
  Thorn configuration files:
  \pause
  \begin{itemize}
    \item \textcolor{blue}{interface.ccl} declares:
      \begin{itemize}
        \item an 'implementation' name
        \item inheritance relationships between thorns
        \item Thorn variables
        \item Global functions, both provided and used
      \end{itemize}
    \pause
    \item \textcolor{blue}{schedule.ccl} declares:
      \begin{itemize}
        \item When the flesh should schedule which functions
        \item When which variables should be allocated/freed
        \item Which variables should be syncronized when
      \end{itemize}
    \pause
    \item \textcolor{blue}{param.ccl} declares:
      \begin{itemize}
        \item Runtime parameters for the thorn
        \item Use/extension of parameters of other thorns
      \end{itemize}
  \end{itemize}
}

\section{HelloWorld Thorn}

\subsection{Standalone Code}

\frame[containsverbatim]{ \frametitle{Writing a Hello World thorn}
We now demonstrate the process of writing a thorn using a simple
Hello World example. \\ \vspace{12pt}

Here is the standalone C code for a Hello World program:
{
\begin{Verbatim}[commandchars=@\[\]]
  @PYaN[#include <stdio.h>]
  @PYaJ[int] main(@PYaJ[void])
  {
    printf("Hello World!\n");
    return @PYaJ[0];
  }
\end{Verbatim}
} }

 \begin{frame}{Creating a New Thorn}
  To Create A New Thorn:
  \begin{columns}[t]
   \column{.6\textwidth}
    \begin{itemize}[<+->]
     \item Select Mojave -- NewThorn from the drop down tab
     \item Title NewThorn and Arrangement as HelloWorld
     \item Fill the other fields as you see fit
    \end{itemize}

  \column{.6\textwidth}
   \begin{centering}
    \begin{figure}[h]
     \centering \includegraphics[height=3.5cm]{NewThorn.png}
    \end{figure}
   \end{centering}
  \end{columns}
 \end{frame}

 \begin{frame}{Creating a New Thorn}
  \begin{columns}[t]
   \column{.5\textwidth}
    From the Fortran Projects column:
     \begin{itemize}[<+->]
      \item Expand arrangements
      \item Expand the HelloWorld arrangement and thorn
      \item Open interface.ccl and schedule.ccl 
     \end{itemize}

  \column{.5\textwidth}
   \begin{centering}
    \begin{figure}[h]
     \centering \includegraphics[width=4cm]{projexp.png}
    \end{figure}
   \end{centering}
  \end{columns}
 \end{frame}

\frame[containsverbatim]{ \frametitle{Hello World Thorn}
Copy the following into each CCL file: \vspace{12pt}
 \begin{itemize}
 \item \verb|interface.ccl|:
  \begin{verbatim}
  implements: HelloWorld\end{verbatim}
 \item \verb|schedule.ccl|:
{
\begin{Verbatim}[commandchars=@\[\]]
  @PYay[schedule] HelloWorld @PYay[at] @PYaL[CCTK_EVOL]
  {
    @PYah[LANG]: C
  } "@PYaB[Print Hello World message]"
\end{Verbatim}
}
 \item \verb|param.ccl|: empty
 \end{itemize}
}

  \begin{frame}{Running Cactus}
   \begin{columns}[t]
   \column{.3\textwidth}
     \begin{centering}
      \begin{figure}[h]
       \centering \includegraphics[height=3.5cm]{newfile.png}
      \end{figure}
     \end{centering}

   \column{.6\textwidth}
    \begin{itemize}[<+->]
     \item Expand the src folder
     \item Right-Click src and select New -- File
     \item Make sure the parent folder is correct
     \item Title the file: HelloWorld.c
    \end{itemize}
   \end{columns}
  \end{frame}


\frame[containsverbatim]{ \frametitle{Hello World Thorn cont.}
 \begin{itemize}
  \item \verb|src/HelloWorld.c|:
{
\begin{Verbatim}[commandchars=@\[\]]
    @PYaN[#include "cctk.h"]
    @PYaN[#include "cctk_Arguments.h"]

    @PYaJ[void] HelloWorld(CCTK_ARGUMENTS)
    {
      DECLARE_CCTK_ARGUMENTS;
      CCTK_INFO("@PYaJ[Hello World!]");
      @PYay[return];
    }
\end{Verbatim}
}
  \item \verb|make.code.defn|:
    \begin{verbatim}
     SRCS = HelloWorld.c\end{verbatim}
 \end{itemize}
}

  \begin{frame}[containsverbatim]{Running Cactus}
       \begin{itemize}
        \item Move to the Cactus/Cactus/par subdirectory 
        \item Create a new file: hello.par
{\item \verb|hello.par|:
\begin{Verbatim}[commandchars=@\[\]]


    ActiveThorns = "HelloWorld"
    Cactus::cctk_itlast = @PYaJ[10]
\end{Verbatim}
}
 \end{itemize}
 \end{frame}


  \begin{frame}[containsverbatim]{Running Cactus}
       \begin{itemize}
        \item Move to the Cactus/Cactus/thornlists subdirectory
        \item Create a new file: hello.th
{\item \verb|hello.th|:
\begin{Verbatim}[commandchars=@\[\]]
    
    
    HelloWorld/HelloWorld
\end{Verbatim}
}
 \end{itemize}
 \end{frame}



 \begin{frame}{Running Cactus}
  \begin{columns}[t]
   \column{.6\textwidth}
   \begin{itemize}[<+->]
    \item Select Mojave from the drop-down tab.
     \item Select Edit Variables
     \item Using the Browse button input:
      \begin{itemize}
       \item hello.par in the par field
       \item helloworld.th in the thornlist field
      \end{itemize}
       \item Input bad2 in the config field
       \item Input HWorld in the sim.name field
       \item Select Save
    \end{itemize}
 
     \column{.3\textwidth}
      \vspace{12pt}
       \begin{centering}
        \begin{figure}[h]
         \centering \includegraphics[width=3.5cm]{editvarib.png}
        \end{figure}
       \end{centering}
    \end{columns}
  \end{frame}


 \begin{frame}{Running HelloWorld}
  \begin{columns}[t]
   \column{.3\textwidth}
    \vspace{12pt}
     \begin{centering}
      \begin{figure}[h]
       \centering \includegraphics[width=2.5cm]{runn.png}
      \end{figure}
     \end{centering}
 
  \column{.6\textwidth}
   \begin{itemize}
    \item To run HelloWorld:
     \begin{itemize}[<+->]
      \item From the Mojave drop down tab:
      \item Select Create
      \item Select Run
     \end{itemize}
   \end{itemize}
  \end{columns}
 \end{frame}

\frame[containsverbatim]{ \frametitle{Hello World Thorn}
 \begin{itemize}
  \item Screen output:\tiny
   \begin{verbatim}       10                                  
  1   0101       ************************  
  01  1010 10      The Cactus Code V4.0    
 1010 1101 011      www.cactuscode.org     
  1001 100101    ************************  
    00010101                               
     100011     (c) Copyright The Authors  
      0100      GNU Licensed. No Warranty  
      0101                                 

Cactus version:    4.0.b17
Compile date:      May 06 2009 (13:15:01)
Run date:          May 06 2009 (13:15:54-0500)
[...]

Activating thorn Cactus...Success -> active implementation Cactus
Activation requested for 
--->HelloWorld<---
Activating thorn HelloWorld...Success -> active implementation HelloWorld
--------------------------------------------------------------------------------
INFO (HelloWorld): Hello World!
INFO (HelloWorld): Hello World!
[...] 8x
--------------------------------------------------------------------------------
Done.
   \end{verbatim}
 \end{itemize}
}

\section{Conway's Game of Life}

\frame{ \frametitle{}
 \begin{centering}
 {\Large Conway's Game of Life}\\
 \end{centering}
}

\subsection{Algorithm}

   \begin{frame} \frametitle{Conway's Game of Life}
    \begin{itemize}
     \item Conway's Game of Life begins with a board of single-celled organisms.
           For each step of the game, a given cell lives if it is surrounded by two or
           three other cells. If a pixel on the board is surrounded by three cells,
           a cell is born on that pixel. If a living cell is surrounded by more than three
           cells, it dies.
     \item \textbf{Algorithm Conway}:
     \begin{enumerate}
      \item Each element in an NxM matrix is initialized to 0 or 1.
      \item For each step of the game, generate a new board as follows: for each pixel 
            on the current board, the value for the pixel for the new board is set as follows:
       \begin{enumerate}
        \item If this pixel contains a 1 and if two or three 1's occupy any of its
              eight neighboring pixels, set the new value to 1.
        \item If this pixel contains a 0 and if three 1's occupy any of its
              eight neighboring pixels, set the new value to 1.
        \item If this pixel contains a 1 and the number of neighboring 1's is less than two
              or more than three, set the new value to 0.
        \item Otherwise set the new value to 0.
       \end{enumerate}
     \end{enumerate}
    \end{itemize}
   \end{frame}

\begin{frame}
 \begin{columns}
  \column{.5\textwidth}
   \justifying Consider the grid to the right.  The cells in the upper left-hand corner
each live because they each have two neighboring cells.  In the upper-right
hand corner, the cell surrounded by the three others dies from overcrowding on
this iteration.  In the lower left-hand corner, the space surrounded by three
others bears a living cell. 

  \column{.5\textwidth}
   \begin{center}
    \includegraphics[width=40mm]{conway-g} \\ \vspace{12pt}
    \includegraphics[width=40mm]{conway-g2} 
   \end{center}
 \end{columns}
\end{frame}

\begin{frame}
  \begin{itemize}[<+->]
   \item The Cactus code for Conway's Game of Life can be found:
   \item Cactus/arrangements/Tutorial/Life/src/life.cc
   \item Notable alterations:
    \begin{itemize}
     \item The CCTK arguements and parameters declarations.
     \item The removal of the print function. (void pboard)
     \item The thorn IOJpeg will be used for output.
    \end{itemize}
  \end{itemize}
   \begin{center}
    \includegraphics[height=40mm]{lifesource.png}
   \end{center}
\end{frame}

\begin{frame}[containsverbatim] \frametitle{Notable Alterations:}
   Additional Header files for communication with Cactus:
 \begin{itemize}
  \item \verb| C code|:
   \tiny
   \begin{Verbatim}

  1 #include <stdlib.h>
  2 #include <unistd.h>
  3 #include <stdio.h>
  4 #include <time.h>


   \end{Verbatim}
  \normalsize
  \item \verb| Cactus code|:
  \tiny
   \begin{Verbatim}[commandchars=@\[\]]

     1  #include <stdlib.h>
     2  #include <unistd.h>
     3  #include <stdio.h>
     4  #include <time.h>
@PYay[     5  #include "cctk.h"                    //three additional headers]
@PYay[     6  #include "cctk_Arguments.h"]
@PYay[     7  #include "cctk_Parameters.h"]
   \end{Verbatim}
  \end{itemize}
\end{frame}

\begin{frame}[containsverbatim] \frametitle{Notable Alterations:}
 Using Cactus' GFINDEX2D and the removal of INDEX2D: 
 \begin{itemize}
  \item \verb| C code|:
   \tiny
   \begin{Verbatim}

32    int cc = INDEX2D(i,j);  

59    int cc = INDEX2D(i,j);




   \end{Verbatim}
  \normalsize
  \item \verb| Cactus code|:
  \tiny
   \begin{Verbatim}[commandchars=@\[\]]

22   int cc = @PYay[CCTK_GFINDEX2D](cctkGH, i, j); 

36   int cc = @PYay[CCTK_GFINDEX2D](cctkGH, i, j);

   \end{Verbatim}
  \end{itemize}
\end{frame}





%\tiny
%\begin{frame}[containsverbatim] \frametitle{Conways Game of Life: C Implementation}
%\verb|life.c|
%\begin{verbatim}
%  1 #include <stdlib.h>
%  2 #include <unistd.h>
%  3 #include <stdio.h>
%  4 #include <time.h>
%  5 
%  6 #define false 0
%  7 #define true 1
%  8 #define naptime 100000
%  9 
% 10 int X=37,Y=37; // The maximum grid size.
% 11 int i,j,di,dj; // Indices for the grid elements.
% 12 
% 13 int main() {
% 14   int **b  = malloc(X*sizeof(int*));   // Memory allocation for this 
% 15   int **nb = malloc(X*sizeof(int*));   // board and the next.
% 16   for (i=0; i<X; i++) {
% 17      b[i]  = calloc(Y,sizeof(int));    // The board elements are 
% 18     nb[i]  = calloc(Y,sizeof(int)); }  // initialized to 0.
% 19 
% 20   int r=0;        // A random number.
% 21   int N = 1000;   // Total number of iterations.
%\end{verbatim}
%\end{frame}
%
%\begin{frame}[containsverbatim] \frametitle{Conways Game of Life: C Implementation}
%\verb|life.c|
%\begin{verbatim}
% 22   int n,s;        // nth iteration and the sum of the neighboring values 
% 23   unsigned int t; // t is a number passed to srand() which seeds
% 24                   // the random-number generator rand().
% 25 
% 26   for (i=0; i<X; i++)
% 27     for (j=0; j<Y; j++) {
% 28        t = (unsigned int)time(NULL)*i*j;
% 29        srand(t); r = rand()%100+1;
% 30        if (r < 10)  b[i][j] = 1; // If the randomly-generated number is less
% 31        else         b[i][j] = 0; // than 10, assign a 1 to this cell.
% 32     }
% 33 
% 34   for (n=1; n<=N; n++) {     // For each iteration of the game,
% 35    usleep((useconds_t)naptime); // Sleep for a few microseconds
% 36    system("clear");             // then clear the screen.
% 37    for (i=0; i<X; i++) {        // Span over the board.
% 38     for (j=0; j<Y; j++)
% 39       if (b[i][j] == 0) printf(" ."); // If this element is a 0, print a
% 40       else              printf(" O"); // dot; otherwise print an O.
% 41     printf("\n"); // End each row with a line terminator.
% 42   }
%\end{verbatim}
%\end{frame}
%
%\begin{frame}[containsverbatim] \frametitle{Conways Game of Life: C Implementation}
%\verb|life.c|
%\begin{verbatim}
% 43     for (i=0; i<X; i++)            // Span over the board. 
% 44       for (j=0; j<Y; j++, s=0) {
% 45         for (di=-1; di<=1; di++)   // Span over the eight pixels 
% 46           for (dj=-1; dj<=1; dj++) // neighboring this one. 
% 47                  if (di  ==0 &&   dj==0) {} // i+0, j+0 is this pixel. 
% 48             else if (i+di <0 || j+dj <0) {} // Outside the left or lower boundary of the grid.
% 49             else if (i+di>=X || j+dj>=Y) {} // Outside the right or upper boundary of the grid.
% 50             else if (b[i+di][j+dj]==1) s++; // If any neighboring element is 1, increment sum.
% 51              if (b[i][j] >  0 && s==2) nb[i][j] = 1; // If alive and sum=2, stay alive.
% 52         else if (b[i][j] >= 0 && s==3) nb[i][j] = 1; // If sum=3, remain alive or be born.
% 53         else if (b[i][j] == 1 && s> 3) nb[i][j] = 0; // If alive and sum>3, die from overcrowding.
% 54         else                           nb[i][j] = 0; // Otherwise remain dead or die.
% 55         // Let each element of this board be equal to that of the next board; blank the next board
% 56         b[i][j] = nb[i][j];
% 57         nb[i][j] = 0;
% 58       }
% 59     printf("\nLast step: %d\n", n);  // Print the nth step.
% 60   }
% 61   printf("\n=)\n\n");
% 62   return 0;
% 63 }
%\end{verbatim}
%\end{frame}
%
%\begin{frame}[containsverbatim] \frametitle{Conways Game of Life: Cactus Implementation}
%\verb|life.c|
%\begin{verbatim}
%  1 #include <stdlib.h>
%  2 #include <unistd.h>
%  3 #include <stdio.h>
%  4 #include <time.h>
%  5 
%  6 const int naptime = 100000; // How long the program will wait before clearing 
%  7                             // the screen and printing a new board.
%  8 
%  9 int i,j,di,dj; // The indices.
% 10 int step=0;    // The nth step. 
% 11 int seed;      // A seed for the random-number generator.
% 12 int points=0;
% 13 
% 14 const int xsiz=30,ysiz=50; // The size of the board. 
% 15 const int nsteps = 335;    // Maximum number of iterations.
% 16 
% 17 typedef int gridfunc[xsiz*ysiz]; // A grid is represented by a 1D array.
% 18 gridfunc b, nb;                  // The current and next boards.
% 19 
% 20 const int INDEX2D(int i,int j) { // For indices i and j, returns the proper index
% 21     return i*ysiz+j;             // of the grid element within the 1D array.
% 22 }
%\end{verbatim}
%\end{frame}
%
%\begin{frame}[containsverbatim] \frametitle{Conways Game of Life: Cactus Implementation}
%\verb|life.c|
%\begin{verbatim}
% 24 void randb(gridfunc &b) { // Produces a random board.
% 25   seed = getpid();        // The seed is equal
% 26   seed = 4570;            // to the process ID.
% 27   srand(seed);            // The RNG is seeded.
% 28   int r;                  // A random number.
% 29   for (i=1; i<xsiz-1; i++)
% 30     for (j=1; j<ysiz-1; j++) {
% 31        r = rand()%100+1;       // Produce a random number.
% 32        int cc = INDEX2D(i,j);  // Find the 1D index.
% 33        if (r < 10)  b[cc] = 1; // If the random number is less than 10,
% 34        else         b[cc] = 0; // let this grid element be a 1; 
% 35                                // otherwise let it be a 0.
% 36     }
% 37 }
% 38 
% 39 // Print a board.
% 40 void pboard(gridfunc &b) {
% 41     usleep((useconds_t)naptime); // Wait for a while
% 42     system("clear");             // then clear the screen.
% 43     for (i=0; i<xsiz; i++) {
% 44         for (j=0; j<ysiz; j++) {
% 45             int cc = INDEX2D(i,j);
% 46             if (b[cc] == 0) printf(" ."); // Span the board; if the element is 0, 
% 47             else            printf(" O"); // print a dot, otherwise print an O.
% 48         }
% 49         printf("\n");
% 50     }
% 51     printf("\nLast step: %d, seed=%d\n", step, seed);
% 52 }
%\end{verbatim}
%\end{frame}
%
%\begin{frame}[containsverbatim] \frametitle{Conways Game of Life: Cactus Implementation}
%\verb|life.c|
%\begin{verbatim}
% 54 void evolve() {
% 55     points = 0; // Set points to 0, initially.
% 56     for (int i=1; i<xsiz-1; i++) {
% 57         for (int j=1; j<ysiz-1; j++) {
% 58             int s=0;               // s is initially 0.
% 59             int cc = INDEX2D(i,j);
% 60             for (int di=-1; di<=1; di++) {
% 61                 for (int dj=-1; dj<=1; dj++) {
% 62                     int cn = INDEX2D(i+di,j+dj);
% 63                     if (di==0 && dj==0) // If this pixel, do nothing.
% 64                         ;
% 65                     else if (b[cn]!=0)  // If the neighboring pixel,
% 66                         s++;            // is alive, increment sum.
% 67                 }
% 68             }
% 69             if (b[cc] >  0 && s==2)      // If the element is alive and the sum is 2,
% 70                 nb[cc] = 1;              // stay alive.
% 71             else if (b[cc] >= 0 && s==3) // If the sum is 3, 
% 72                 nb[cc] = 1;              // remain alive or be born.
% 73             else if (b[cc] == 1 && s> 3) // If the element is alive and the sum is greater than 3,
% 74                 nb[cc] = 0;              // the element dies from overcrowding.
% 75             else                         // Otherwise 
% 76                 nb[cc] = 0;              // the element dies or remains dead.
% 77         }
% 78     }
%\end{verbatim}
%\end{frame}
%
%\begin{frame}[containsverbatim] \frametitle{Conways Game of Life: Cactus Implementation}
%\verb|life.c|
%\begin{verbatim}
% 79     for (int i=0;i<xsiz;i++) {
% 80         for(int j=0;j<ysiz;j++) {
% 81             int cc = INDEX2D(i,j); // Set each element of this board to the corresponding 
% 82             b[cc] = nb[cc];        // element of the new board. 
% 83             if(nb[cc])
% 84                 points++;          // A point is earned for each living cell.
% 85         }
% 86     }
% 87 }
% 88 
% 89 int main() {
% 90     randb(b);
% 91     for (step=1; step<=nsteps; step++) { // Iterate for nsteps.
% 92         pboard(b);
% 93         evolve();
% 94         if(points == 0) break;           // If every cell is dead, stop.
% 95     }
% 96     pboard(b);
% 97     printf(" =) \n");
% 98     return 0;
% 99 }
%\end{verbatim}
%\end{frame}
% 
%\normalsize

\subsection{Implementation}

   \begin{frame}[containsverbatim] \frametitle{Definition Files}
    \begin{itemize}
     \item \verb|interface.ccl|:
     \tiny\begin{verbatim}
# Interface definition for thorn Life
implements: life
inherits:

CCTK_INT boards TYPE=gf
{
        b, nb
} "boards"
\end{verbatim}
   \normalsize \vspace{12pt}
    \item \verb|param.ccl|:
     \tiny\begin{verbatim}
# Parameter definitions for thorn Life

CCTK_INT random_seed "Initializer for the random number generator"
{
        *:* :: "Determines the starting position for the game of life"
} 0
\end{verbatim}
    \end{itemize}
   \end{frame}

%   \begin{frame} \frametitle{Conways Game of Life: Cactus Implementation}
%    \begin{itemize}
%    \item Things to note in the source code:
%     \begin{itemize}
%      \item The additional includes required for communication with Cactus.
%      \item The use of Cactus's GFINDEX2D and the subsequent removal of INDEX2D.
%      \item The removal of the print function. Another Cactus thorn (IOJpeg) will be used for visualization.
%      \item The CCTK Parameters declartion in the random board method; required by use of the seed variable 
%            from param.ccl.
%     \end{itemize}
%    \end{itemize}
%   \end{frame}
%
%   \begin{frame}[containsverbatim]  \frametitle{Conways Game of Life: Thorn Source Code}
%    \verb|life.cc|
%     \tiny \begin{verbatim}
%     1  #include <stdlib.h>
%     2  #include <unistd.h>
%     3  #include <stdio.h>
%     4  #include <time.h>
%     5  #include "cctk.h"                                               //three additional headers
%     6  #include "cctk_Arguments.h"
%     7  #include "cctk_Parameters.h"
%     8
%     9  const int naptime = 100000;
%    10
%    11  extern "C" void life_randb(CCTK_ARGUMENTS);                     //methods must be declared;
%    12  extern "C" void life_evolve(CCTK_ARGUMENTS);                    //and later scheduled
%    13
%    14  void life_randb(CCTK_ARGUMENTS) {
%    15          DECLARE_CCTK_ARGUMENTS;d
%    16          DECLARE_CCTK_PARAMETERS;                                //parameter declartion necessary
%    17          srand(random_seed);                                     //for seed variable
%    18          int r = 0;
%    19          for (int i = 1; i < cctk_lsh[0] - 1; i++) {
%    20                  for (int j = 1; j < cctk_lsh[1] - 1; j++) {
%    21                          r = rand() % 100 + 1;
%    22                          int cc = CCTK_GFINDEX2D(cctkGH, i, j);  //using a built in Cactus function
%    23                          if (r < 10)                            
%    24                                  b[cc] = 1;
%    25                          else
%    26                                  b[cc] = 0;
%    27                  }
%    28          }
%    29  }
%    \end{verbatim}
% \end{frame}
%
%    \begin{frame}[containsverbatim]  \frametitle{Conways Game of Life: Thorn Source Code}
%    \verb|life.cc|
%     \tiny \begin{verbatim}
%    31  void life_evolve(CCTK_ARGUMENTS) {
%    32          DECLARE_CCTK_ARGUMENTS;
%    33          for (int i = 1; i < cctk_lsh[0] - 1; i++) {
%    34                  for (int j = 1; j < cctk_lsh[1] - 1; j++) {
%    35                          int s = 0;
%    36                          int cc = CCTK_GFINDEX2D(cctkGH, i, j);
%    37                          for (int di = -1; di <= 1; di++) {
%    38                                  for (int dj = -1; dj <= 1; dj++) {
%    39                                          int cn = CCTK_GFINDEX2D(cctkGH, i + di, j + dj);
%    40                                          if (di == 0 && dj == 0)
%    41                                                  ;
%    42                                          else if (b[cn] != 0)
%    43                                                  s++;
%    44                                  }
%    45                          }
%    46                          if (b[cc] > 0 && s == 2)
%    47                                  nb[cc] = 1;
%    48                          else if (b[cc] >= 0 && s == 3)
%    49                                  nb[cc] = 1;
%    50                          else if (b[cc] == 1 && s > 3)
%    51                                  nb[cc] = 0;
%    52                          else
%    53                                  nb[cc] = 0;
%    54                  }
%    55          }
%    56          for (int i = 0; i < cctk_lsh[0]; i++) {
%    57                  for (int j = 0; j < cctk_lsh[1]; j++) {
%    58                          int cc = CCTK_GFINDEX2D(cctkGH, i, j);
%    59                          b[cc] = nb[cc]; }      }             }
%    \end{verbatim}
% \end{frame}

   \begin{frame}[containsverbatim] \frametitle{Conways Game of Life: Cactus Implementation}
    \begin{itemize}
    \item make.code.defn:
    \tiny\begin{verbatim}
# Main make.code.defn file for thorn Life

# Source files in this directory
SRCS = life.cc

# Subdirectories containing source files
SUBDIRS =
\end{verbatim}
    \normalsize \vspace{6pt}
    \item schedule.ccl:
    \tiny\begin{verbatim}
# Schedule definitions for thorn Life

storage: boards

schedule life_randb at INITIAL
{
        LANG: C
        SYNC: boards
} "setup board"

schedule life_evolve at EVOL
{
        LANG: C
        SYNC: boards
} "another generation lives or dies"
\end{verbatim}
    \normalsize
    \end{itemize}
   \end{frame}

  \begin{frame}{Running Cactus}
   \begin{columns}[t]
   \column{.6\textwidth}
      To run Conway's Life through Mojave:
       \begin{itemize}[<+->]
        \item Select Edit Variables from the Mojave tab
        \item For the par option:
         \begin{itemize}
          \item Select Browse -- Cactus -- par -- life2.par
         \end{itemize}
        \item For the config  option:
         \begin{itemize}
          \item Input: bad
         \end{itemize}
        \item Name the simulation life2
        \item Mojave-Create and Mojave-Run
       \end{itemize}

   \column{.3\textwidth}
    \begin{centering}
     \begin{figure}[h]
      \centering \includegraphics[height=3.5cm]{varib.png}
     \end{figure}
    \end{centering}
  \end{columns}
 \end{frame}

\begin{frame}[containsverbatim] \frametitle{Visualizing Conway's Life Results}
 \begin{itemize}
  \item After running Conway's Life, you may navigate to the
\verb|simulations/life2/output-*| directory to see the JPEG 
image output.
  \item To animate the images, use the ImageMagick command
\verb|animate|. Since your VM uses minimal RAM, you may wish
to animate only a fraction of the images:
  \item \verb|animate "_[0-9][0-9]\."|
  \item This will animate images 11-99. 
 \end{itemize}
\end{frame}

\section{Parallelization}

\frame{ \frametitle{}
 \begin{centering}
 {\Large Parallelization Framework in Cactus}\\
 \end{centering}
}

\subsection{The Driver Thorn}

\frame[containsverbatim]{ \frametitle{The Driver}
  \begin{itemize}
    \item Special thorn
    \item Only one active for a run, choose at starttime
    \item The only code (almost) dealing with parallelism
    \item Other thorns access parallelism via API
    \item Underlying parallel layer transparent to application thorns
    \item Examples
     \begin{itemize}
      \item PUGH: unigrid, part of the Cactus computational toolkit
      \item Carpet: mesh refinement, \verb|http://www.carpetcode.org|
     \end{itemize}
  \end{itemize}
}

\subsection{Data Distribution}

\frame { \frametitle{Grid functions}
 Cactus provides methods to:
 \begin{itemize}
  \item Distribute variables across processes (grid function)
  \item Synchronize processor domain boundaries between processes
  \item Compute reductions across grid functions
  \item Actual implementation in driver thorn
 \end{itemize}
}

\frame { \frametitle{Ghost Zones}
 \begin{itemize}
  \item Grid variables: distributed across processes
  \item Assumption: Most work done (quasi-) locally:\\
        True for hyperbolic and parabolic problems
  \item Split of computational domain into blocks
  \item Only communication at the borders (faces)
  \item At least stencil size many ghostzones
 \end{itemize}
}

\frame { \frametitle{Ghost Zone Example}
 Without Ghostzones:\\
 \begin{centering}\includegraphics[width=9cm]{1dnoghost}\\\end{centering}
 With Ghostzones:\\
 \begin{centering}\includegraphics[width=9cm]{withghost}\\\end{centering}
}

\frame { \frametitle{Domain Decomposition}
 \begin{centering}\includegraphics[width=11cm]{domain_decomposition}\\\end{centering}
}

\frame { \frametitle{Mesh Refinement Decomposition}
 \begin{centering}\includegraphics[width=11cm]{domain_decomposition_mr}\\\end{centering}
}


\section{Wave Equation}

\frame{ \frametitle{}
 \begin{centering}
 {\Large Solving the Wave Equation}\\
 \end{centering}
}

\subsection{Formulation}

\begin{frame} \frametitle{The Wave Equation}
 \begin{itemize}
 \item PDE which describes wave propagation in a medium
 \item one of the fundamental equations of mathematical physics
 \item \emph{ For a spatial domain $\mathcal{D}$, find a 
 scalar field $\psi(x,y,z,t)$ which satisfies:}
 \end{itemize}
 \begin{columns}[t]
    \column{.4\textwidth}
      \begin{figure}[!htp]
      \begin{center}
      \begin{tikzpicture}[rotate=5,scale=1.2]
        \filldraw[fill=green!60!white, draw=green!50!black, thick]
             (-1,-1) rectangle (1,1);
        \draw (0,0) node {{\large $\mathcal{D}$}};
        \draw (1.4,0.2) node {$\partial\mathcal{D}$};
        \draw[->,thick,color=red] (1.1,0.2) -- (1,0.2);
      \end{tikzpicture}
      \end{center}
      \end{figure}

    \column{.6\textwidth}
      \begin{align*}
      &\frac{\partial^2\psi}{\partial t^2} = c^2\nabla^2\psi
      &
      \;\;\;
      &\textit{inside $\mathcal{D}$, and}
      \\
      \\
      &B(\psi,\partial_i\psi)|_{\partial\mathcal{D}} = 0
      &
      \;\;\;
      &\textit{at $\partial\mathcal{D}$}.
      \end{align*}
      \emph{where $c$ is the speed of the wave,\linebreak
      $B(\psi,\partial_i\psi)$ specifies the boundary conditions, and
      $\nabla^2$ is the \emph{Laplacian} of $\psi$:}
      {\small
      \begin{align*}
        \nabla^2\psi \equiv
          \left[
            \frac{\partial^2}{\partial x^2}
          + \frac{\partial^2}{\partial y^2}
          + \frac{\partial^2}{\partial z^2}
          \right] \psi
      \end{align*}}
 \end{columns}
\end{frame}

\begin{frame} \frametitle{First-order form}
 The wave equation is a second-order PDE for a single scalar function.
 To improve its numerical stability properties, we can rewrite it as a
 system of first-order PDEs:
  \begin{align*}
   \frac{\partial \psi}{\partial t}    &= c\vec{\nabla} \cdot \vec{p} &
   \frac{\partial \vec{p}}{\partial t} &= c\nabla\psi 
  \end{align*}
  with a new unknown vector variable $\vec{p}$. If $\vec{p}=\nabla\psi$,
  we recover the original scalar wave equation.

 \vspace{3mm}
 We are going to solve this system of PDEs inside a \emph{cube}
 $\mathcal{D}=[-a,a]^3$ with the following boundary conditions:
 \begin{align*}
   \psi|_{\partial\mathcal{D}} &= 0
   &
   \vec{p}_\parallel|_{\partial\mathcal{D}} &= 0
   &
   \vec{p}_\perp|_{\partial\mathcal{D}_{i+}} &=
   \vec{p}_\perp|_{\partial\mathcal{D}_{i-}} 
 \end{align*}
 where $\vec{p}_\perp$, $\vec{p}_\parallel$ are the (outward) normal and
 tangential components of $\vec{p}$, and {\small
 $\partial\mathcal{D}_{i+}/\partial\mathcal{D}_{i-}$} are the opposite
 faces normal to the $i$-th axis.
\end{frame}

\subsection{Discretization}

\begin{frame}
\frametitle{Discretization}
 We will be solving the wave equation using finite difference (FD)
 methods on rectangular grids (adding mesh refinement later).
 \begin{columns}[t]
    \column{.4\textwidth}
      \begin{figure}[!htp]
      \begin{center}
      \only<1>{
      \begin{tikzpicture}[rotate=5,scale=1.2]
        \filldraw[fill=green!60!white, draw=green!50!black, thick]
             (-1,-1) rectangle (1,1);
        \draw (0,0) node {{\large $\mathcal{D}$}};
      \end{tikzpicture}
      } %only1
      \only<2>{
      \begin{tikzpicture}[rotate=5,scale=1.2]
        \draw[draw=green!50!black, thick]
             (-1,-1) rectangle (1,1);
        \draw[-,thin,color=green!50!black] (-0.8,-1) -- (-0.8,1);
        \draw[-,thin,color=green!50!black] (-0.6,-1) -- (-0.6,1);
        \draw[-,thin,color=green!50!black] (-0.5,-0.5) -- (-0.5,0.5);
        \draw[-,thin,color=green!50!black] (-0.4,-1) -- (-0.4,1);
        \draw[-,thin,color=green!50!black] (-0.3,-0.5) -- (-0.3,0.5);
        \draw[-,thin,color=green!50!black] (-0.2,-1) -- (-0.2,1);
        \draw[-,thin,color=green!50!black] (-0.1,-0.5) -- (-0.1,0.5);
        \draw[-,thin,color=green!50!black] (0.8,-1) -- (0.8,1);
        \draw[-,thin,color=green!50!black] (0.6,-1) -- (0.6,1);
        \draw[-,thin,color=green!50!black] (0.5,-0.5) -- (0.5,0.5);
        \draw[-,thin,color=green!50!black] (0.4,-1) -- (0.4,1);
        \draw[-,thin,color=green!50!black] (0.3,-0.5) -- (0.3,0.5);
        \draw[-,thin,color=green!50!black] (0.2,-1) -- (0.2,1);
        \draw[-,thin,color=green!50!black] (0.1,-0.5) -- (0.1,0.5);
        \draw[-,thin,color=green!50!black] ( 0.0,-1) -- ( 0.0,1);
        \draw[-,thin,color=green!50!black] ( -1,0.0) -- (1,0.0);
        \draw[-,thin,color=green!50!black] (-1,-0.8) -- (1,-0.8);
        \draw[-,thin,color=green!50!black] (-1,-0.6) -- (1,-0.6);
        \draw[-,thin,color=green!50!black] (-0.5,-0.5) -- (0.5,-0.5);
        \draw[-,thin,color=green!50!black] (-1,-0.4) -- (1,-0.4);
        \draw[-,thin,color=green!50!black] (-0.5,-0.3) -- (0.5,-0.3);
        \draw[-,thin,color=green!50!black] (-1,-0.2) -- (1,-0.2);
        \draw[-,thin,color=green!50!black] (-0.5,-0.1) -- (0.5,-0.1);
        \draw[-,thin,color=green!50!black] (-1,0.8) -- (1,0.8);
        \draw[-,thin,color=green!50!black] (-1,0.6) -- (1,0.6);
        \draw[-,thin,color=green!50!black] (-0.5,0.5) -- (0.5,0.5);
        \draw[-,thin,color=green!50!black] (-1,0.4) -- (1,0.4);
        \draw[-,thin,color=green!50!black] (-0.5,0.3) -- (0.5,0.3);
        \draw[-,thin,color=green!50!black] (-1,0.2) -- (1,0.2);
        \draw[-,thin,color=green!50!black] (-0.5,0.1) -- (0.5,0.1);
        \filldraw[fill=white, draw=white, thin]
             (-0.2,-0.2) rectangle (0.2,0.2);
        \draw (0,0) node {{\large $\mathcal{D}$}};
      \end{tikzpicture}
      } %only1
      \end{center}
      \end{figure}

    \column{.6\textwidth}
      \begin{itemize}
      \item spatial domain $\to$ rectangular grid
      \item continuous function $\to$ grid function
      \item partial derivative $\to$ FD operator
      \item PDE $\to$ system of algebraic equations
      \end{itemize}
  \end{columns}

  \begin{align*}
   \begin{array}{l}
    \frac{\partial \psi}{\partial t}    = c\vec{\nabla} \cdot \vec{p} \\
    \\
    \frac{\partial \vec{p}}{\partial t} = c\nabla\psi \\
   \end{array}
   \Big | \to \Big |
   \only<1>{
   \begin{array}{ll}
    D_t\psi = K_\psi := c (D_x p_x + D_y p_y + D_z p_z) \\
    D_t p_x = K_x    := c D_x\psi                       \\
    D_t p_y = K_y    := c D_y\psi                       \\
    D_t p_z = K_z    := c D_z\psi                       \\
   \end{array}
   }
   \only<2->{
   \textcolor{blue}{
   \begin{array}{ll}
    D_t\psi = K_\psi := c (D_x p_x + D_y p_y + D_z p_z) \\
    D_t p_x = K_x    := c D_x\psi                       \\
    D_t p_y = K_y    := c D_y\psi                       \\
    D_t p_z = K_z    := c D_z\psi                       \\
   \end{array}}
   }
  \end{align*}

  where $D_t$, $D_x$, $D_y$ and $D_z$ are the \emph{finite differencing}
  operators.

\end{frame}

\begin{frame} 
\frametitle{Spatial derivatives}
  We approximate spatial derivatives by finite differences
  of the grid function values at neighboring points. We will use
  a centered scheme, which is second-order accurate in grid spacing 
  $\Delta$:
  \begin{align*}
    (D_x \psi)_{i,j,k} &= \frac{\psi_{i+1,j,k}-\psi_{i-1,j,k}}{2\Delta} \\
    (D_y \psi)_{i,j,k} &= \frac{\psi_{i,j+1,k}-\psi_{i,j-1,k}}{2\Delta} \\
    (D_z \psi)_{i,j,k} &= \frac{\psi_{i,j,k+1}-\psi_{i,j,k-1}}{2\Delta} \\
  \end{align*}
\end{frame}

\begin{frame} 
\frametitle{Runge-Kutta time integration}
  Simplest Runge-Kutta method: Newton integration (1st order accurate)
  \begin{align*}
    \frac{y_{n+1}-y_n}{\Delta t} &= K_y(t_n,y_n) 
    &
    \to
    &
    &y_{n+1} = y_n + K_y(t_n, y_n)\Delta t
  \end{align*}

  We will use a 2nd order accurate Runge-Kutta integration scheme (also known 
  as a \emph{Heun method}) with two intermediate steps:
  \begin{align*}
    &k_1 = K_y(t_n, y_n) \\
    &\tilde{y}_n = y_n + k_1\Delta t, \\
    &k_2 = K_y(t_n + \Delta t, \tilde{y}_n) \\
    &y_{n+1} = y_n + \left(\frac{1}{2} k_1 + \frac{1}{2} k_2\right)\Delta t \\
  \end{align*}
\end{frame}

\begin{frame} \frametitle{Discrete boundary conditions}
  We place the grid points in such a way that the boundary of the
  domain $\partial\mathcal{D}$ lies in between the two last grid points:
  \begin{columns}
  \column{.7\textwidth}
    \begin{center}
      \includegraphics[width=7cm]{badwave_bcs}\\
    \end{center}

  \column{.3\textwidth}
    \begin{align*}
      \psi|_{\partial\mathcal{D}} &= 0 \\
      \vec{p}_\parallel|_{\partial\mathcal{D}} &= 0 \\
      \vec{p}_\perp|_{\partial\mathcal{D}_{i+}} &=
      \vec{p}_\perp|_{\partial\mathcal{D}_{i-}} \\
      &\Downarrow
      \\
      \psi_{0,j,k}&=-\psi_{1,j,k}, \\
      \vec{p}_{y,0,j,k} &=-\vec{p}_{y,1,j,k}, \\
      \vec{p}_{z,0,j,k} &=-\vec{p}_{z,1,j,k}, \\
      \vec{p}_{x,0,j,k} &= \vec{p}_{x,1,j,k}, 
    \end{align*}
    \emph{... etc.}

  \end{columns}
\end{frame}

\subsection{Standalone Code}

\begin{frame}[containsverbatim] \frametitle{Standalone code}
\begin{columns}
\column{.7\textwidth}
{\small
\begin{Verbatim}[commandchars=@\[\]]
@PYaJ[int] @PYaL[main](@PYaJ[int] argc,@PYaJ[char] @PYbe[*]@PYbe[*]argv) {
    @PYaJ[FILE] @PYbe[*]fp @PYbe[=] fopen(@PYaB["]@PYaB[psi.d.asc]@PYaB["],@PYaB["]@PYaB[w+]@PYaB["]);
    BadWave_Init();
    BadWave_ApplyBounds();
    BadWave_print(fp);
    @PYaE[// evolve 100 steps]
    @PYay[for](@PYaJ[int] i@PYbe[=]@PYag[0];i@PYbe[<]@PYag[100];i@PYbe[+]@PYbe[+]) {
        iter @PYbe[=] i;
        BadWave_Evolve();
        BadWave_ApplyBounds();
        BadWave_Evolve2();
        BadWave_TmpApplyBounds();
        BadWave_print(fp);
    }
    fclose(fp);
    @PYay[return] @PYag[0];
}
\end{Verbatim}
}%
\column{.3\textwidth}
  \includegraphics[width=.7\textwidth]{schedule_tree}\\
\end{columns}
\end{frame}

\begin{frame}[containsverbatim] \frametitle{Standalone code: parameters and definitions}
{\small
\begin{Verbatim}[commandchars=@\[\]]
@PYaE[// Parameters: ]
@PYaJ[bool] verbose @PYbe[=] @PYaX[true];     @PYaE[// enable verbose output]
@PYaJ[double] wave_speed @PYbe[=] @PYaw[1.0]; @PYaE[// wave speed factor]
@PYay[const] @PYaJ[int] isiz @PYbe[=] @PYag[32], jsiz @PYbe[=] @PYag[32], ksiz @PYbe[=] @PYag[32]; @PYaE[//grid size]

@PYaE[// 1D array to hold the grid and a function to find 3D indices]
@PYay[typedef] @PYaJ[double] gridfunc@PYZlb[]isiz@PYbe[*]jsiz@PYbe[*]ksiz@PYZrb[];
@PYap[inline] @PYaJ[int] @PYaL[INDEX3D](@PYaJ[int] i,@PYaJ[int] j,@PYaJ[int] k) { @PYay[return] (i@PYbe[+]j@PYbe[*]isiz)@PYbe[*]ksiz@PYbe[+]k; }

@PYaE[// Grid functions:]
@PYaJ[gridfunc] psi,px,py,pz;           @PYaE[// psi and p]
@PYaJ[gridfunc] tpsi,tpx,tpy,tpz;       @PYaE[// temporary variables]
@PYaJ[gridfunc] kpsi,kpx,kpy,kpz;       @PYaE[// the kernels]
@PYaJ[gridfunc] k2psi,k2px,k2py,k2pz;

@PYaE[// Other global variables: iteration, grid spacing and time step]
@PYaJ[int] iter@PYbe[=]@PYag[0];    @PYaJ[double] dx, dy, dz, dt; 
\end{Verbatim}
} %small
\end{frame}

\subsection{Thorn BadWave}

\begin{frame}[containsverbatim] \frametitle{Writing thorn BadWave: \tt{param.ccl}}
\begin{Verbatim}[commandchars=@\[\]]
@PYaN[#]@PYaN[ Parameter definitions for thorn BadWave]

@PYah[private:]
@PYaJ[CCTK_REAL] wave_speed @PYaB["]@PYaB[characteristic speed at boundary]@PYaB["] 
{
  @PYaB["]@PYaB[0.1:*]@PYaB["] @PYbe[:]@PYbe[:] @PYaB["]@PYaB[wave speed]@PYaB["]
} @PYaw[1.]

@PYah[private:]
@PYaJ[CCTK_BOOLEAN] verbose @PYaB["]@PYaB[whether to print stuff]@PYaB["]
{
    @PYbe[:]@PYbe[:] @PYaB["]@PYaB[verbose flag]@PYaB["]
} @PYag[0]
\end{Verbatim}
\end{frame}

\begin{frame}[containsverbatim] \frametitle{Writing thorn BadWave: \tt{interface.ccl}}
\begin{Verbatim}[commandchars=@\[\]]
@PYaN[#]@PYaN[ Interface definition for thorn BadWave]
@PYah[implements:] BadWaveTutorial
@PYah[inherits:] grid

@PYaJ[CCTK_REAL] wave_vars TYPE@PYbe[=]gf
{
    psi
    px, py, pz
} @PYaB["]@PYaB[Basic components of wave equation code]@PYaB["]

@PYaJ[CCTK_REAL] tmp_wave_vars TYPE@PYbe[=]gf { tpsi, tpx, tpy, tpz } <...>
@PYaJ[CCTK_REAL] kernels TYPE@PYbe[=]gf { 
    kpsi kpx, kpy, kpz, k2psi, k2px, k2py, k2pz
} <...>

\end{Verbatim}
\end{frame}

\begin{frame}[containsverbatim] \frametitle{Writing thorn BadWave: \tt{schedule.ccl}}
{\small
\begin{Verbatim}[commandchars=@\[\]]
@PYaN[#]@PYaN[ Schedule definitions for thorn BadWave]
@PYah[storage:] wave_vars, tmp_wave_vars, kernels

@PYay[schedule] BadWave_Init @PYay[at] @PYaL[INITIAL]
{ @PYah[LANG:] C
} @PYaB["]@PYaB[Startup and initialize]@PYaB["]

@PYay[schedule] BadWave_ApplyBounds @PYay[at] @PYaL[POSTSTEP]
{   @PYah[LANG:] C
    @PYah[SYNC:] wave_vars
} @PYaB["]@PYaB[Apply boundary conditions]@PYaB["]

@PYay[schedule] BadWave_Evolve @PYay[at] @PYaL[EVOL] <...>
@PYay[schedule] BadWave_TmpApplyBounds @PYay[at] @PYaL[EVOL] @PYay[after] BadWave_Evolve <...>
@PYay[schedule] BadWave_Evolve2 @PYay[at] @PYaL[EVOL] @PYay[after] BadWave_TmpApplyBounds <...>
\end{Verbatim}
} %small
\end{frame}

\begin{frame}[containsverbatim] \frametitle{Adapting the source code for Cactus}
Modifications to standalone code:
\begin{itemize}
  \item include Cactus headers:
{\small
\begin{Verbatim}[commandchars=@\[\]]
    @PYaN[#]@PYaN[include "cctk.h"]
    @PYaN[#]@PYaN[include "cctk_Arguments.h"]
    @PYaN[#]@PYaN[include "cctk_Parameters.h"]
\end{Verbatim}
} %small
  \item declare all scheduled functions with external C linkage:
{\small
\begin{Verbatim}[commandchars=@\[\]]
    @PYay[extern] @PYaB["]@PYaB[C]@PYaB["] @PYaJ[void] BadWave_Init(CCTK_ARGUMENTS);           
    @PYay[extern] @PYaB["]@PYaB[C]@PYaB["] @PYaJ[void] BadWave_ApplyBounds(CCTK_ARGUMENTS);    
    @PYay[extern] @PYaB["]@PYaB[C]@PYaB["] @PYaJ[void] BadWave_TmpApplyBounds(CCTK_ARGUMENTS); 
    @PYay[extern] @PYaB["]@PYaB[C]@PYaB["] @PYaJ[void] BadWave_Evolve(CCTK_ARGUMENTS);         
    @PYay[extern] @PYaB["]@PYaB[C]@PYaB["] @PYaJ[void] BadWave_Evolve2(CCTK_ARGUMENTS);
\end{Verbatim}
} %small

  \item add the {\tt CCTK\_ARGUMENTS} macro to scheduled functions:
{\small
\begin{Verbatim}[commandchars=@\[\]]
    @PYaJ[void] @PYaL[BadWave_Init](CCTK_ARGUMENTS) 
    {
        DECLARE_CCTK_ARGUMENTS; 
        ...
    }
\end{Verbatim}
} %small
        this tells functions which part of the grid they will be working on.
\end{itemize}
\end{frame}

\begin{frame}[containsverbatim] \frametitle{Grid definitions in Cactus}
\begin{columns}
\column{.4\textwidth}
   \begin{center}
      \includegraphics[width=\textwidth]{cctk_grid_defns}\\
   \end{center}

\column{.6\textwidth}
   \begin{itemize}
   \item {\tt cctk\_gsh}:  global grid dimensions
   \item {\tt cctk\_lsh}:  local grid dimensions
   \item {\tt cctk\_delta\_space}: grid spacing
   \item {\tt cctk\_nghostzones}:  number of ghostzones
   \item {\tt cctk\_bbox}: whether a boundary is an outer one
   \item {\tt CCTK\_GFINDEX3D(i,j,k)}: computes local 1D grid index
   \end{itemize}
\end{columns}

\end{frame}

\subsection{Running BadWave}

\begin{frame}[containsverbatim] \frametitle{Simple parameter file}
Parfile: {\tt par/BadWavePUGH.par}
\begin{itemize}
\item include the driver thorn (PUGH)
\item specify grid size
\item specify BadWave parameters
\end{itemize}

\begin{Verbatim}[commandchars=@\[\]]
ActiveThorns @PYbe[=] @PYaB["]@PYaB[BadWave PUGH]@PYaB["]

cactus@PYbe[:]@PYbe[:]cctk_itlast  @PYbe[=] @PYag[100]
pugh@PYbe[:]@PYbe[:]global_nsize   @PYbe[=] @PYag[32]
pugh@PYbe[:]@PYbe[:]ghost_size     @PYbe[=] @PYag[1]

BadWavePUGH@PYbe[:]@PYbe[:]wave_speed @PYbe[=] @PYaw[2.0]
\end{Verbatim}

\end{frame}

\begin{frame}[containsverbatim] \frametitle{More advanced parameter file}
Parfile: {\tt par/BadWavePUGHv2.par}

Includes new thorns:
\begin{itemize}
\item {\tt CoordBase}:  coordinate extents
\item {\tt CardGrid3D}: provides variety of grid configurations
\item {\tt SymBase}:    basic thorn for specifying symmetries of the domain
\item {\tt IOBasic, IOUtil, IOScalar, IOASCII}: basic and advanced I/O
\item {\tt Time}: provides global time and iteration variables
\item {\tt PUGHSlab, PUGHReduce, LocalReduce}: required by I/O thorns
\end{itemize}

\end{frame}

\subsection{Method of Lines}

\begin{frame} \frametitle{Method of Lines}

\begin{itemize}
\item Method of lines is a general name for an approach of solving PDEs with time 
variable, in which the PDE is approximated by a system of interdependent
ODEs for each grid point. 
\item Method of lines allows to extend time integration methods developed for ODEs 
to PDEs.
\item Cactus thorn MoL implements several different time integration methods.
\item We can use these methods if we modify our thorn: {\tt BadWaveMoL}
\end{itemize}
\end{frame}


\begin{frame}[containsverbatim] \frametitle{Registering evolved variables with MoL}

\begin{itemize}
\item in Wave.cc:
{\small
\begin{Verbatim}[commandchars=@\[\]]
@PYaJ[void] @PYaL[BadWaveMoL_Register](CCTK_ARGUMENTS)
{
    DECLARE_CCTK_ARGUMENTS;
    MoLRegisterEvolved(CCTK_VarIndex(@PYaB["]@PYaB[BadWaveMoL::psi]@PYaB["]),
                       CCTK_VarIndex(@PYaB["]@PYaB[BadWaveMoL::kpsi]@PYaB["]));
    ...
}
\end{Verbatim}
} %small

\item in {\tt schedule.ccl}:
\begin{Verbatim}[commandchars=@\[\]]
@PYah[storage:] wave_vars@PYZlb[]@PYag[3]@PYZrb[], kernels@PYZlb[]@PYag[3]@PYZrb[]
schedule BadWaveMoL_Register in MoL_Register
{
    @PYah[LANG:] C
} @PYaB["]@PYaB[Register for MoL]@PYaB["]
\end{Verbatim}
\end{itemize}

\end{frame}

\begin{frame}[containsverbatim] \frametitle{Registering evolved variables with MoL}
\begin{itemize}
\item in {\tt interface.ccl}:
{\small
\begin{Verbatim}[commandchars=@\[\]]
CCTK_INT FUNCTION MoLRegisterEvolved \
  (CCTK_INT IN EvolvedIndex, CCTK_INT IN RHSIndex)
USES FUNCTION MoLRegisterEvolved
\end{Verbatim}
}

\item in the parfile ({\tt par/BadWaveMoL.par}):
\begin{verbatim}
ActiveThorns = "BadWaveMoL carpet
CarpetIOBasic CarpetIOASCII CarpetIOScalar CarpetLib LoopControl CoordBase
IOUtil SymBase Time CarpetReduce CartGrid3D MoL"

MoL::ODE_Method = "RK4"
MoL::MoL_Intermediate_Steps = 4
MoL::MoL_Num_Scratch_Levels = 1
\end{verbatim}
\end{itemize}
\end{frame}

\section{Mesh Refinement}

\subsection{Fixed mesh refinement}

\begin{frame}[containsverbatim] \frametitle{Berger-Oliger mesh refinement}
\begin{columns}
\column{0.5\textwidth}
  \begin{itemize}
  \item "Box-in-box" refinement:
    \begin{center}
       \includegraphics[width=0.6\textwidth]{cactus_fmr}
    \end{center}
  \item Time update:
    \begin{center}
       \includegraphics[width=0.6\textwidth]{cactus_fmr_update}
    \end{center}
  \end{itemize}
\column{0.5\textwidth}
  {\footnotesize
    \begin{minipage}[t]{0.8\textwidth}
      \include{CBO}
    \end{minipage}
  } %footnotesize
\end{columns}
\end{frame}

\begin{frame}[containsverbatim] \frametitle{Fixed mesh refinement}
Parfile: {\tt par/BadWaveFMR.par}

{\small
\begin{Verbatim}[commandchars=@\[\]]
ActiveThorns @PYbe[=] @PYaB["]@PYaB[CarpetRegrid2 Dissipation]
@PYaB[SpaceMask SphericalSurface]@PYaB["]

CarpetRegrid2@PYbe[:]@PYbe[:]num_levels_1 @PYbe[=] @PYag[2]
Carpet@PYbe[:]@PYbe[:]max_refinement_levels @PYbe[=] @PYag[2]
CarpetRegrid2@PYbe[:]@PYbe[:]num_centres @PYbe[=] @PYag[1]
CarpetRegrid2@PYbe[:]@PYbe[:]Position_x_1 @PYbe[=] @PYaw[0.5]
CarpetRegrid2@PYbe[:]@PYbe[:]Position_y_1 @PYbe[=] @PYaw[0.5]
CarpetRegrid2@PYbe[:]@PYbe[:]Position_z_1 @PYbe[=] @PYaw[0.5]
CarpetRegrid2@PYbe[:]@PYbe[:]radius_1@PYZlb[]@PYag[1]@PYZrb[] @PYbe[=] @PYaw[0.1]
@PYaN[#]@PYaN[CarpetRegrid2::radius_1@PYZlb[]2@PYZrb[] = 0.1]
CarpetRegrid2@PYbe[:]@PYbe[:]verbose @PYbe[=] @PYaB["]@PYaB[yes]@PYaB["]
Carpet@PYbe[:]@PYbe[:]init_fill_timelevels@PYbe[=]@PYaB["]@PYaB[yes]@PYaB["]
Dissipation@PYbe[:]@PYbe[:]vars @PYbe[=] @PYaB["]@PYaB[BadWaveMoL::psi]@PYaB["]
\end{Verbatim}
}
\end{frame}

\subsection{Adaptive mesh refinement}

\begin{frame}[containsverbatim] \frametitle{Adaptive mesh refinement}
Modifications to the code:
{\footnotesize
\begin{Verbatim}[commandchars=@\[\]]
@PYaJ[void] @PYaL[BadWaveAMR_BoxMover](CCTK_ARGUMENTS)
{
    DECLARE_CCTK_ARGUMENTS;
    @PYaJ[int] max_refinement_levels @PYbe[=] @PYag[30];

    radius@PYZlb[]max_refinement_levels@PYbe[*]@PYag[0]@PYbe[+]@PYag[1]@PYZrb[]@PYbe[=]@PYaw[.05];

    @PYaE[// Make the box wiggle]
    position_x@PYZlb[]@PYag[0]@PYZrb[] @PYbe[=] position_y@PYZlb[]@PYag[0]@PYZrb[] @PYbe[=] position_z@PYZlb[]@PYag[0]@PYZrb[] @PYbe[=] @PYaw[0.7]@PYbe[+]@PYaw[0.05]@PYbe[*]sin(cctk_time);
    active@PYZlb[]@PYag[0]@PYZrb[]@PYbe[=]@PYag[1];
    num_levels@PYZlb[]@PYag[0]@PYZrb[]@PYbe[=]@PYag[2]; @PYaE[// two levels: 0=base grid, 1=first refined grid]

    @PYaE[// Turn on the next box]
    active@PYZlb[]@PYag[1]@PYZrb[] @PYbe[=] @PYag[1];
    radius@PYZlb[]max_refinement_levels@PYbe[*]@PYag[1]@PYbe[+]@PYag[1]@PYZrb[]@PYbe[=]@PYaw[.05];
    position_x@PYZlb[]@PYag[1]@PYZrb[]@PYbe[=]position_y@PYZlb[]@PYag[1]@PYZrb[]@PYbe[=]position_z@PYZlb[]@PYag[1]@PYZrb[]@PYbe[=]@PYaw[.3];
    num_levels@PYZlb[]@PYag[1]@PYZrb[]@PYbe[=]@PYag[2];
}
\end{Verbatim}
}
\end{frame}

\begin{frame}[containsverbatim] \frametitle{Adaptive mesh refinement}
\begin{itemize}
\item Modifications to the {\tt schedule.ccl}:
\begin{Verbatim}[commandchars=@\[\]]
schedule BadWaveAMR_BoxMover at preregrid
{
    @PYah[LANG:] C
} @PYaB["]@PYaB[Jiggle the box]@PYaB["]
\end{Verbatim}
\item Modifications to the {\tt interface.ccl}:
\begin{verbatim}
   inherits: grid, CarpetRegrid2
\end{verbatim}
\end{itemize}
\end{frame}

\begin{frame}[containsverbatim] \frametitle{Adaptive mesh refinement}
Modifications in the parfile ({\tt pars/BadWaveAMR.par}):
{\small
\begin{Verbatim}[commandchars=@\[\]]
CarpetRegrid2@PYbe[:]@PYbe[:]regrid_every @PYbe[=] @PYag[2]
Carpet@PYbe[:]@PYbe[:]max_refinement_levels @PYbe[=] @PYag[3]
CarpetRegrid2@PYbe[:]@PYbe[:]num_centres @PYbe[=] @PYag[2]

CarpetRegrid2@PYbe[:]@PYbe[:]num_levels_1 @PYbe[=] @PYag[3]
CarpetRegrid2@PYbe[:]@PYbe[:]Position_x_1 @PYbe[=] @PYaw[0.3]
CarpetRegrid2@PYbe[:]@PYbe[:]Position_y_1 @PYbe[=] @PYaw[0.3]
CarpetRegrid2@PYbe[:]@PYbe[:]Position_z_1 @PYbe[=] @PYaw[0.3]
CarpetRegrid2@PYbe[:]@PYbe[:]radius_1@PYZlb[]@PYag[1]@PYZrb[] @PYbe[=] @PYaw[0.2]
CarpetRegrid2@PYbe[:]@PYbe[:]radius_1@PYZlb[]@PYag[2]@PYZrb[] @PYbe[=] @PYaw[0.1]

CarpetRegrid2@PYbe[:]@PYbe[:]num_levels_2 @PYbe[=] @PYag[2]
CarpetRegrid2@PYbe[:]@PYbe[:]Position_x_2 @PYbe[=] @PYaw[0.7]
CarpetRegrid2@PYbe[:]@PYbe[:]Position_y_2 @PYbe[=] @PYaw[0.7]
CarpetRegrid2@PYbe[:]@PYbe[:]Position_z_2 @PYbe[=] @PYaw[0.7]
CarpetRegrid2@PYbe[:]@PYbe[:]radius_2@PYZlb[]@PYag[1]@PYZrb[] @PYbe[=] @PYaw[0.05]
\end{Verbatim}
}
\end{frame}

\begin{frame}[containsverbatim] \frametitle{Adaptive mesh refinement}
Modifications in the parfile ({\tt pars/BadWaveAMR.par}):
{\footnotesize
\begin{Verbatim}[commandchars=@\[\]]
ActiveThorns @PYbe[=] @PYaB["]@PYaB[IOJPeg CarpetSlab CarpetInterp ]
         @PYaB[CarpetInterp2 AEILocalInterp]@PYaB["]
IOJpeg@PYbe[:]@PYbe[:]out_every               @PYbe[=] @PYag[1]
IOJpeg@PYbe[:]@PYbe[:]out_vars                @PYbe[=] @PYaB["]@PYaB[BadWaveAMR::psi]@PYaB["]

IOJPeg@PYbe[:]@PYbe[:]gridpoints @PYbe[=] interpolate
IOJpeg@PYbe[:]@PYbe[:]array2d_x0         @PYbe[=] @PYaw[0.0]
IOJpeg@PYbe[:]@PYbe[:]array2d_y0         @PYbe[=] @PYaw[0.0]
IOJpeg@PYbe[:]@PYbe[:]array2d_z0         @PYbe[=]  @PYaw[0.5]
IOJpeg@PYbe[:]@PYbe[:]array2d_npoints_i  @PYbe[=] @PYag[101]
IOJpeg@PYbe[:]@PYbe[:]array2d_dx_i       @PYbe[=]  @PYaw[0.01]
IOJpeg@PYbe[:]@PYbe[:]array2d_dy_i       @PYbe[=]  @PYag[0]
IOJpeg@PYbe[:]@PYbe[:]array2d_dz_i       @PYbe[=]  @PYag[0]
IOJpeg@PYbe[:]@PYbe[:]array2d_npoints_j  @PYbe[=] @PYag[101]
IOJpeg@PYbe[:]@PYbe[:]array2d_dx_j       @PYbe[=]  @PYag[0]
IOJpeg@PYbe[:]@PYbe[:]array2d_dy_j       @PYbe[=]  @PYaw[0.01]
IOJpeg@PYbe[:]@PYbe[:]array2d_dz_j       @PYbe[=]  @PYag[0]
\end{Verbatim}
}
\end{frame}



