\input{pygmentsheader} \section{Thorn Structure} \frame{ \frametitle{} \hspace{1em} \begin{centering} \hspace{-1cm}\includegraphics[height=3.3cm]{cactuslogo_big}\\ \vspace{1cm} {\Huge Writing Cactus Thorns}\\ \end{centering} } \frame{ \frametitle{} {\Large Plan: \begin{itemize} \item Thorn structure \item HelloWorld thorn \item Conway's Game of Life \begin{itemize} \item Standalone code \item The \tt{Life} thorn \end{itemize} \item Solving the Wave Equation \begin{itemize} \item Standalone code \item The BadWave thorn \item Advanced thorns: PUGH, MoL, AMR \end{itemize} \end{itemize}} } \subsection{Thorn Structure} \frame { \frametitle{Thorn Structure} Inside view of a plug-in module, or thorn for Cactus \begin{figure} \vspace{0.0cm} \hspace{0.0cm} \centering \includegraphics[height=0.55\textwidth]{thorn_arch} \end{figure} } \frame { \frametitle{Thorn Specification} Thorn configuration files: \pause \begin{itemize} \item \textcolor{blue}{interface.ccl} declares: \begin{itemize} \item an 'implementation' name \item inheritance relationships between thorns \item Thorn variables \item Global functions, both provided and used \end{itemize} \pause \item \textcolor{blue}{schedule.ccl} declares: \begin{itemize} \item When the flesh should schedule which functions \item When which variables should be allocated/freed \item Which variables should be syncronized when \end{itemize} \pause \item \textcolor{blue}{param.ccl} declares: \begin{itemize} \item Runtime parameters for the thorn \item Use/extension of parameters of other thorns \end{itemize} \end{itemize} } \section{HelloWorld Thorn} \subsection{Standalone Code} \frame[containsverbatim]{ \frametitle{Writing a Hello World thorn} We now demonstrate the process of writing a thorn using a simple Hello World example. \\ \vspace{12pt} Here is the standalone C code for a Hello World program: { \begin{Verbatim}[commandchars=@\[\]] @PYaN[#include ] @PYaJ[int] main(@PYaJ[void]) { printf("Hello World!\n"); return @PYaJ[0]; } \end{Verbatim} } } \begin{frame}{Creating a New Thorn} To Create A New Thorn: \begin{columns}[t] \column{.6\textwidth} \begin{itemize}[<+->] \item Select Mojave -- NewThorn from the drop down tab \item Title NewThorn and Arrangement as HelloWorld \item Fill the other fields as you see fit \end{itemize} \column{.6\textwidth} \begin{centering} \begin{figure}[h] \centering \includegraphics[height=3.5cm]{NewThorn.png} \end{figure} \end{centering} \end{columns} \end{frame} \begin{frame}{Creating a New Thorn} \begin{columns}[t] \column{.5\textwidth} From the Fortran Projects column: \begin{itemize}[<+->] \item Expand arrangements \item Expand the HelloWorld arrangement and thorn \item Open interface.ccl and schedule.ccl \end{itemize} \column{.5\textwidth} \begin{centering} \begin{figure}[h] \centering \includegraphics[width=4cm]{projexp.png} \end{figure} \end{centering} \end{columns} \end{frame} \frame[containsverbatim]{ \frametitle{Hello World Thorn} Copy the following into each CCL file: \vspace{12pt} \begin{itemize} \item \verb|interface.ccl|: \begin{verbatim} implements: HelloWorld\end{verbatim} \item \verb|schedule.ccl|: { \begin{Verbatim}[commandchars=@\[\]] @PYay[schedule] HelloWorld @PYay[at] @PYaL[CCTK_EVOL] { @PYah[LANG]: C } "@PYaB[Print Hello World message]" \end{Verbatim} } \item \verb|param.ccl|: empty \end{itemize} } \begin{frame}{Running Cactus} \begin{columns}[t] \column{.3\textwidth} \begin{centering} \begin{figure}[h] \centering \includegraphics[height=3.5cm]{newfile.png} \end{figure} \end{centering} \column{.6\textwidth} \begin{itemize}[<+->] \item Expand the src folder \item Right-Click src and select New -- File \item Make sure the parent folder is correct \item Title the file: HelloWorld.c \end{itemize} \end{columns} \end{frame} \frame[containsverbatim]{ \frametitle{Hello World Thorn cont.} \begin{itemize} \item \verb|src/HelloWorld.c|: { \begin{Verbatim}[commandchars=@\[\]] @PYaN[#include "cctk.h"] @PYaN[#include "cctk_Arguments.h"] @PYaJ[void] HelloWorld(CCTK_ARGUMENTS) { DECLARE_CCTK_ARGUMENTS; CCTK_INFO("@PYaJ[Hello World!]"); @PYay[return]; } \end{Verbatim} } \item \verb|make.code.defn|: \begin{verbatim} SRCS = HelloWorld.c\end{verbatim} \end{itemize} } \begin{frame}[containsverbatim]{Running Cactus} \begin{itemize} \item Move to the Cactus/Cactus/par subdirectory \item Create a new file: hello.par {\item \verb|hello.par|: \begin{Verbatim}[commandchars=@\[\]] ActiveThorns = "HelloWorld" Cactus::cctk_itlast = @PYaJ[10] \end{Verbatim} } \end{itemize} \end{frame} \begin{frame}[containsverbatim]{Running Cactus} \begin{itemize} \item Move to the Cactus/Cactus/thornlists subdirectory \item Create a new file: hello.th {\item \verb|hello.th|: \begin{Verbatim}[commandchars=@\[\]] HelloWorld/HelloWorld \end{Verbatim} } \end{itemize} \end{frame} \begin{frame}{Running Cactus} \begin{columns}[t] \column{.6\textwidth} \begin{itemize}[<+->] \item Select Mojave from the drop-down tab. \item Select Edit Variables \item Using the Browse button input: \begin{itemize} \item hello.par in the par field \item helloworld.th in the thornlist field \end{itemize} \item Input bad2 in the config field \item Input HWorld in the sim.name field \item Select Save \end{itemize} \column{.3\textwidth} \vspace{12pt} \begin{centering} \begin{figure}[h] \centering \includegraphics[width=3.5cm]{editvarib.png} \end{figure} \end{centering} \end{columns} \end{frame} \begin{frame}{Running HelloWorld} \begin{columns}[t] \column{.3\textwidth} \vspace{12pt} \begin{centering} \begin{figure}[h] \centering \includegraphics[width=2.5cm]{runn.png} \end{figure} \end{centering} \column{.6\textwidth} \begin{itemize} \item To run HelloWorld: \begin{itemize}[<+->] \item From the Mojave drop down tab: \item Select Create \item Select Run \end{itemize} \end{itemize} \end{columns} \end{frame} \frame[containsverbatim]{ \frametitle{Hello World Thorn} \begin{itemize} \item Screen output:\tiny \begin{verbatim} 10 1 0101 ************************ 01 1010 10 The Cactus Code V4.0 1010 1101 011 www.cactuscode.org 1001 100101 ************************ 00010101 100011 (c) Copyright The Authors 0100 GNU Licensed. No Warranty 0101 Cactus version: 4.0.b17 Compile date: May 06 2009 (13:15:01) Run date: May 06 2009 (13:15:54-0500) [...] Activating thorn Cactus...Success -> active implementation Cactus Activation requested for --->HelloWorld<--- Activating thorn HelloWorld...Success -> active implementation HelloWorld -------------------------------------------------------------------------------- INFO (HelloWorld): Hello World! INFO (HelloWorld): Hello World! [...] 8x -------------------------------------------------------------------------------- Done. \end{verbatim} \end{itemize} } \section{Conway's Game of Life} \frame{ \frametitle{} \begin{centering} {\Large Conway's Game of Life}\\ \end{centering} } \subsection{Algorithm} \begin{frame} \frametitle{Conway's Game of Life} \begin{itemize} \item Conway's Game of Life begins with a board of single-celled organisms. For each step of the game, a given cell lives if it is surrounded by two or three other cells. If a pixel on the board is surrounded by three cells, a cell is born on that pixel. If a living cell is surrounded by more than three cells, it dies. \item \textbf{Algorithm Conway}: \begin{enumerate} \item Each element in an NxM matrix is initialized to 0 or 1. \item For each step of the game, generate a new board as follows: for each pixel on the current board, the value for the pixel for the new board is set as follows: \begin{enumerate} \item If this pixel contains a 1 and if two or three 1's occupy any of its eight neighboring pixels, set the new value to 1. \item If this pixel contains a 0 and if three 1's occupy any of its eight neighboring pixels, set the new value to 1. \item If this pixel contains a 1 and the number of neighboring 1's is less than two or more than three, set the new value to 0. \item Otherwise set the new value to 0. \end{enumerate} \end{enumerate} \end{itemize} \end{frame} \begin{frame} \begin{columns} \column{.5\textwidth} \justifying Consider the grid to the right. The cells in the upper left-hand corner each live because they each have two neighboring cells. In the upper-right hand corner, the cell surrounded by the three others dies from overcrowding on this iteration. In the lower left-hand corner, the space surrounded by three others bears a living cell. \column{.5\textwidth} \begin{center} \includegraphics[width=40mm]{conway-g} \\ \vspace{12pt} \includegraphics[width=40mm]{conway-g2} \end{center} \end{columns} \end{frame} \begin{frame} \begin{itemize}[<+->] \item The Cactus code for Conway's Game of Life can be found: \item Cactus/arrangements/Tutorial/Life/src/life.cc \item Notable alterations: \begin{itemize} \item The CCTK arguements and parameters declarations. \item The removal of the print function. (void pboard) \item The thorn IOJpeg will be used for output. \end{itemize} \end{itemize} \begin{center} \includegraphics[height=40mm]{lifesource.png} \end{center} \end{frame} \begin{frame}[containsverbatim] \frametitle{Notable Alterations:} Additional Header files for communication with Cactus: \begin{itemize} \item \verb| C code|: \tiny \begin{Verbatim} 1 #include 2 #include 3 #include 4 #include \end{Verbatim} \normalsize \item \verb| Cactus code|: \tiny \begin{Verbatim}[commandchars=@\[\]] 1 #include 2 #include 3 #include 4 #include @PYay[ 5 #include "cctk.h" //three additional headers] @PYay[ 6 #include "cctk_Arguments.h"] @PYay[ 7 #include "cctk_Parameters.h"] \end{Verbatim} \end{itemize} \end{frame} \begin{frame}[containsverbatim] \frametitle{Notable Alterations:} Using Cactus' GFINDEX2D and the removal of INDEX2D: \begin{itemize} \item \verb| C code|: \tiny \begin{Verbatim} 32 int cc = INDEX2D(i,j); 59 int cc = INDEX2D(i,j); \end{Verbatim} \normalsize \item \verb| Cactus code|: \tiny \begin{Verbatim}[commandchars=@\[\]] 22 int cc = @PYay[CCTK_GFINDEX2D](cctkGH, i, j); 36 int cc = @PYay[CCTK_GFINDEX2D](cctkGH, i, j); \end{Verbatim} \end{itemize} \end{frame} %\tiny %\begin{frame}[containsverbatim] \frametitle{Conways Game of Life: C Implementation} %\verb|life.c| %\begin{verbatim} % 1 #include % 2 #include % 3 #include % 4 #include % 5 % 6 #define false 0 % 7 #define true 1 % 8 #define naptime 100000 % 9 % 10 int X=37,Y=37; // The maximum grid size. % 11 int i,j,di,dj; // Indices for the grid elements. % 12 % 13 int main() { % 14 int **b = malloc(X*sizeof(int*)); // Memory allocation for this % 15 int **nb = malloc(X*sizeof(int*)); // board and the next. % 16 for (i=0; i=X || j+dj>=Y) {} // Outside the right or upper boundary of the grid. % 50 else if (b[i+di][j+dj]==1) s++; // If any neighboring element is 1, increment sum. % 51 if (b[i][j] > 0 && s==2) nb[i][j] = 1; // If alive and sum=2, stay alive. % 52 else if (b[i][j] >= 0 && s==3) nb[i][j] = 1; // If sum=3, remain alive or be born. % 53 else if (b[i][j] == 1 && s> 3) nb[i][j] = 0; // If alive and sum>3, die from overcrowding. % 54 else nb[i][j] = 0; // Otherwise remain dead or die. % 55 // Let each element of this board be equal to that of the next board; blank the next board % 56 b[i][j] = nb[i][j]; % 57 nb[i][j] = 0; % 58 } % 59 printf("\nLast step: %d\n", n); // Print the nth step. % 60 } % 61 printf("\n=)\n\n"); % 62 return 0; % 63 } %\end{verbatim} %\end{frame} % %\begin{frame}[containsverbatim] \frametitle{Conways Game of Life: Cactus Implementation} %\verb|life.c| %\begin{verbatim} % 1 #include % 2 #include % 3 #include % 4 #include % 5 % 6 const int naptime = 100000; // How long the program will wait before clearing % 7 // the screen and printing a new board. % 8 % 9 int i,j,di,dj; // The indices. % 10 int step=0; // The nth step. % 11 int seed; // A seed for the random-number generator. % 12 int points=0; % 13 % 14 const int xsiz=30,ysiz=50; // The size of the board. % 15 const int nsteps = 335; // Maximum number of iterations. % 16 % 17 typedef int gridfunc[xsiz*ysiz]; // A grid is represented by a 1D array. % 18 gridfunc b, nb; // The current and next boards. % 19 % 20 const int INDEX2D(int i,int j) { // For indices i and j, returns the proper index % 21 return i*ysiz+j; // of the grid element within the 1D array. % 22 } %\end{verbatim} %\end{frame} % %\begin{frame}[containsverbatim] \frametitle{Conways Game of Life: Cactus Implementation} %\verb|life.c| %\begin{verbatim} % 24 void randb(gridfunc &b) { // Produces a random board. % 25 seed = getpid(); // The seed is equal % 26 seed = 4570; // to the process ID. % 27 srand(seed); // The RNG is seeded. % 28 int r; // A random number. % 29 for (i=1; i 0 && s==2) // If the element is alive and the sum is 2, % 70 nb[cc] = 1; // stay alive. % 71 else if (b[cc] >= 0 && s==3) // If the sum is 3, % 72 nb[cc] = 1; // remain alive or be born. % 73 else if (b[cc] == 1 && s> 3) // If the element is alive and the sum is greater than 3, % 74 nb[cc] = 0; // the element dies from overcrowding. % 75 else // Otherwise % 76 nb[cc] = 0; // the element dies or remains dead. % 77 } % 78 } %\end{verbatim} %\end{frame} % %\begin{frame}[containsverbatim] \frametitle{Conways Game of Life: Cactus Implementation} %\verb|life.c| %\begin{verbatim} % 79 for (int i=0;i % 2 #include % 3 #include % 4 #include % 5 #include "cctk.h" //three additional headers % 6 #include "cctk_Arguments.h" % 7 #include "cctk_Parameters.h" % 8 % 9 const int naptime = 100000; % 10 % 11 extern "C" void life_randb(CCTK_ARGUMENTS); //methods must be declared; % 12 extern "C" void life_evolve(CCTK_ARGUMENTS); //and later scheduled % 13 % 14 void life_randb(CCTK_ARGUMENTS) { % 15 DECLARE_CCTK_ARGUMENTS;d % 16 DECLARE_CCTK_PARAMETERS; //parameter declartion necessary % 17 srand(random_seed); //for seed variable % 18 int r = 0; % 19 for (int i = 1; i < cctk_lsh[0] - 1; i++) { % 20 for (int j = 1; j < cctk_lsh[1] - 1; j++) { % 21 r = rand() % 100 + 1; % 22 int cc = CCTK_GFINDEX2D(cctkGH, i, j); //using a built in Cactus function % 23 if (r < 10) % 24 b[cc] = 1; % 25 else % 26 b[cc] = 0; % 27 } % 28 } % 29 } % \end{verbatim} % \end{frame} % % \begin{frame}[containsverbatim] \frametitle{Conways Game of Life: Thorn Source Code} % \verb|life.cc| % \tiny \begin{verbatim} % 31 void life_evolve(CCTK_ARGUMENTS) { % 32 DECLARE_CCTK_ARGUMENTS; % 33 for (int i = 1; i < cctk_lsh[0] - 1; i++) { % 34 for (int j = 1; j < cctk_lsh[1] - 1; j++) { % 35 int s = 0; % 36 int cc = CCTK_GFINDEX2D(cctkGH, i, j); % 37 for (int di = -1; di <= 1; di++) { % 38 for (int dj = -1; dj <= 1; dj++) { % 39 int cn = CCTK_GFINDEX2D(cctkGH, i + di, j + dj); % 40 if (di == 0 && dj == 0) % 41 ; % 42 else if (b[cn] != 0) % 43 s++; % 44 } % 45 } % 46 if (b[cc] > 0 && s == 2) % 47 nb[cc] = 1; % 48 else if (b[cc] >= 0 && s == 3) % 49 nb[cc] = 1; % 50 else if (b[cc] == 1 && s > 3) % 51 nb[cc] = 0; % 52 else % 53 nb[cc] = 0; % 54 } % 55 } % 56 for (int i = 0; i < cctk_lsh[0]; i++) { % 57 for (int j = 0; j < cctk_lsh[1]; j++) { % 58 int cc = CCTK_GFINDEX2D(cctkGH, i, j); % 59 b[cc] = nb[cc]; } } } % \end{verbatim} % \end{frame} \begin{frame}[containsverbatim] \frametitle{Conways Game of Life: Cactus Implementation} \begin{itemize} \item make.code.defn: \tiny\begin{verbatim} # Main make.code.defn file for thorn Life # Source files in this directory SRCS = life.cc # Subdirectories containing source files SUBDIRS = \end{verbatim} \normalsize \vspace{6pt} \item schedule.ccl: \tiny\begin{verbatim} # Schedule definitions for thorn Life storage: boards schedule life_randb at INITIAL { LANG: C SYNC: boards } "setup board" schedule life_evolve at EVOL { LANG: C SYNC: boards } "another generation lives or dies" \end{verbatim} \normalsize \end{itemize} \end{frame} \begin{frame}{Running Cactus} \begin{columns}[t] \column{.6\textwidth} To run Conway's Life through Mojave: \begin{itemize}[<+->] \item Select Edit Variables from the Mojave tab \item For the par option: \begin{itemize} \item Select Browse -- Cactus -- par -- life2.par \end{itemize} \item For the config option: \begin{itemize} \item Input: bad \end{itemize} \item Name the simulation life2 \item Mojave-Create and Mojave-Run \end{itemize} \column{.3\textwidth} \begin{centering} \begin{figure}[h] \centering \includegraphics[height=3.5cm]{varib.png} \end{figure} \end{centering} \end{columns} \end{frame} \begin{frame}[containsverbatim] \frametitle{Visualizing Conway's Life Results} \begin{itemize} \item After running Conway's Life, you may navigate to the \verb|simulations/life2/output-*| directory to see the JPEG image output. \item To animate the images, use the ImageMagick command \verb|animate|. Since your VM uses minimal RAM, you may wish to animate only a fraction of the images: \item \verb|animate "_[0-9][0-9]\."| \item This will animate images 11-99. \end{itemize} \end{frame} \section{Parallelization} \frame{ \frametitle{} \begin{centering} {\Large Parallelization Framework in Cactus}\\ \end{centering} } \subsection{The Driver Thorn} \frame[containsverbatim]{ \frametitle{The Driver} \begin{itemize} \item Special thorn \item Only one active for a run, choose at starttime \item The only code (almost) dealing with parallelism \item Other thorns access parallelism via API \item Underlying parallel layer transparent to application thorns \item Examples \begin{itemize} \item PUGH: unigrid, part of the Cactus computational toolkit \item Carpet: mesh refinement, \verb|http://www.carpetcode.org| \end{itemize} \end{itemize} } \subsection{Data Distribution} \frame { \frametitle{Grid functions} Cactus provides methods to: \begin{itemize} \item Distribute variables across processes (grid function) \item Synchronize processor domain boundaries between processes \item Compute reductions across grid functions \item Actual implementation in driver thorn \end{itemize} } \frame { \frametitle{Ghost Zones} \begin{itemize} \item Grid variables: distributed across processes \item Assumption: Most work done (quasi-) locally:\\ True for hyperbolic and parabolic problems \item Split of computational domain into blocks \item Only communication at the borders (faces) \item At least stencil size many ghostzones \end{itemize} } \frame { \frametitle{Ghost Zone Example} Without Ghostzones:\\ \begin{centering}\includegraphics[width=9cm]{1dnoghost}\\\end{centering} With Ghostzones:\\ \begin{centering}\includegraphics[width=9cm]{withghost}\\\end{centering} } \frame { \frametitle{Domain Decomposition} \begin{centering}\includegraphics[width=11cm]{domain_decomposition}\\\end{centering} } \frame { \frametitle{Mesh Refinement Decomposition} \begin{centering}\includegraphics[width=11cm]{domain_decomposition_mr}\\\end{centering} } \section{Wave Equation} \frame{ \frametitle{} \begin{centering} {\Large Solving the Wave Equation}\\ \end{centering} } \subsection{Formulation} \begin{frame} \frametitle{The Wave Equation} \begin{itemize} \item PDE which describes wave propagation in a medium \item one of the fundamental equations of mathematical physics \item \emph{ For a spatial domain $\mathcal{D}$, find a scalar field $\psi(x,y,z,t)$ which satisfies:} \end{itemize} \begin{columns}[t] \column{.4\textwidth} \begin{figure}[!htp] \begin{center} \begin{tikzpicture}[rotate=5,scale=1.2] \filldraw[fill=green!60!white, draw=green!50!black, thick] (-1,-1) rectangle (1,1); \draw (0,0) node {{\large $\mathcal{D}$}}; \draw (1.4,0.2) node {$\partial\mathcal{D}$}; \draw[->,thick,color=red] (1.1,0.2) -- (1,0.2); \end{tikzpicture} \end{center} \end{figure} \column{.6\textwidth} \begin{align*} &\frac{\partial^2\psi}{\partial t^2} = c^2\nabla^2\psi & \;\;\; &\textit{inside $\mathcal{D}$, and} \\ \\ &B(\psi,\partial_i\psi)|_{\partial\mathcal{D}} = 0 & \;\;\; &\textit{at $\partial\mathcal{D}$}. \end{align*} \emph{where $c$ is the speed of the wave,\linebreak $B(\psi,\partial_i\psi)$ specifies the boundary conditions, and $\nabla^2$ is the \emph{Laplacian} of $\psi$:} {\small \begin{align*} \nabla^2\psi \equiv \left[ \frac{\partial^2}{\partial x^2} + \frac{\partial^2}{\partial y^2} + \frac{\partial^2}{\partial z^2} \right] \psi \end{align*}} \end{columns} \end{frame} \begin{frame} \frametitle{First-order form} The wave equation is a second-order PDE for a single scalar function. To improve its numerical stability properties, we can rewrite it as a system of first-order PDEs: \begin{align*} \frac{\partial \psi}{\partial t} &= c\vec{\nabla} \cdot \vec{p} & \frac{\partial \vec{p}}{\partial t} &= c\nabla\psi \end{align*} with a new unknown vector variable $\vec{p}$. If $\vec{p}=\nabla\psi$, we recover the original scalar wave equation. \vspace{3mm} We are going to solve this system of PDEs inside a \emph{cube} $\mathcal{D}=[-a,a]^3$ with the following boundary conditions: \begin{align*} \psi|_{\partial\mathcal{D}} &= 0 & \vec{p}_\parallel|_{\partial\mathcal{D}} &= 0 & \vec{p}_\perp|_{\partial\mathcal{D}_{i+}} &= \vec{p}_\perp|_{\partial\mathcal{D}_{i-}} \end{align*} where $\vec{p}_\perp$, $\vec{p}_\parallel$ are the (outward) normal and tangential components of $\vec{p}$, and {\small $\partial\mathcal{D}_{i+}/\partial\mathcal{D}_{i-}$} are the opposite faces normal to the $i$-th axis. \end{frame} \subsection{Discretization} \begin{frame} \frametitle{Discretization} We will be solving the wave equation using finite difference (FD) methods on rectangular grids (adding mesh refinement later). \begin{columns}[t] \column{.4\textwidth} \begin{figure}[!htp] \begin{center} \only<1>{ \begin{tikzpicture}[rotate=5,scale=1.2] \filldraw[fill=green!60!white, draw=green!50!black, thick] (-1,-1) rectangle (1,1); \draw (0,0) node {{\large $\mathcal{D}$}}; \end{tikzpicture} } %only1 \only<2>{ \begin{tikzpicture}[rotate=5,scale=1.2] \draw[draw=green!50!black, thick] (-1,-1) rectangle (1,1); \draw[-,thin,color=green!50!black] (-0.8,-1) -- (-0.8,1); \draw[-,thin,color=green!50!black] (-0.6,-1) -- (-0.6,1); \draw[-,thin,color=green!50!black] (-0.5,-0.5) -- (-0.5,0.5); \draw[-,thin,color=green!50!black] (-0.4,-1) -- (-0.4,1); \draw[-,thin,color=green!50!black] (-0.3,-0.5) -- (-0.3,0.5); \draw[-,thin,color=green!50!black] (-0.2,-1) -- (-0.2,1); \draw[-,thin,color=green!50!black] (-0.1,-0.5) -- (-0.1,0.5); \draw[-,thin,color=green!50!black] (0.8,-1) -- (0.8,1); \draw[-,thin,color=green!50!black] (0.6,-1) -- (0.6,1); \draw[-,thin,color=green!50!black] (0.5,-0.5) -- (0.5,0.5); \draw[-,thin,color=green!50!black] (0.4,-1) -- (0.4,1); \draw[-,thin,color=green!50!black] (0.3,-0.5) -- (0.3,0.5); \draw[-,thin,color=green!50!black] (0.2,-1) -- (0.2,1); \draw[-,thin,color=green!50!black] (0.1,-0.5) -- (0.1,0.5); \draw[-,thin,color=green!50!black] ( 0.0,-1) -- ( 0.0,1); \draw[-,thin,color=green!50!black] ( -1,0.0) -- (1,0.0); \draw[-,thin,color=green!50!black] (-1,-0.8) -- (1,-0.8); \draw[-,thin,color=green!50!black] (-1,-0.6) -- (1,-0.6); \draw[-,thin,color=green!50!black] (-0.5,-0.5) -- (0.5,-0.5); \draw[-,thin,color=green!50!black] (-1,-0.4) -- (1,-0.4); \draw[-,thin,color=green!50!black] (-0.5,-0.3) -- (0.5,-0.3); \draw[-,thin,color=green!50!black] (-1,-0.2) -- (1,-0.2); \draw[-,thin,color=green!50!black] (-0.5,-0.1) -- (0.5,-0.1); \draw[-,thin,color=green!50!black] (-1,0.8) -- (1,0.8); \draw[-,thin,color=green!50!black] (-1,0.6) -- (1,0.6); \draw[-,thin,color=green!50!black] (-0.5,0.5) -- (0.5,0.5); \draw[-,thin,color=green!50!black] (-1,0.4) -- (1,0.4); \draw[-,thin,color=green!50!black] (-0.5,0.3) -- (0.5,0.3); \draw[-,thin,color=green!50!black] (-1,0.2) -- (1,0.2); \draw[-,thin,color=green!50!black] (-0.5,0.1) -- (0.5,0.1); \filldraw[fill=white, draw=white, thin] (-0.2,-0.2) rectangle (0.2,0.2); \draw (0,0) node {{\large $\mathcal{D}$}}; \end{tikzpicture} } %only1 \end{center} \end{figure} \column{.6\textwidth} \begin{itemize} \item spatial domain $\to$ rectangular grid \item continuous function $\to$ grid function \item partial derivative $\to$ FD operator \item PDE $\to$ system of algebraic equations \end{itemize} \end{columns} \begin{align*} \begin{array}{l} \frac{\partial \psi}{\partial t} = c\vec{\nabla} \cdot \vec{p} \\ \\ \frac{\partial \vec{p}}{\partial t} = c\nabla\psi \\ \end{array} \Big | \to \Big | \only<1>{ \begin{array}{ll} D_t\psi = K_\psi := c (D_x p_x + D_y p_y + D_z p_z) \\ D_t p_x = K_x := c D_x\psi \\ D_t p_y = K_y := c D_y\psi \\ D_t p_z = K_z := c D_z\psi \\ \end{array} } \only<2->{ \textcolor{blue}{ \begin{array}{ll} D_t\psi = K_\psi := c (D_x p_x + D_y p_y + D_z p_z) \\ D_t p_x = K_x := c D_x\psi \\ D_t p_y = K_y := c D_y\psi \\ D_t p_z = K_z := c D_z\psi \\ \end{array}} } \end{align*} where $D_t$, $D_x$, $D_y$ and $D_z$ are the \emph{finite differencing} operators. \end{frame} \begin{frame} \frametitle{Spatial derivatives} We approximate spatial derivatives by finite differences of the grid function values at neighboring points. We will use a centered scheme, which is second-order accurate in grid spacing $\Delta$: \begin{align*} (D_x \psi)_{i,j,k} &= \frac{\psi_{i+1,j,k}-\psi_{i-1,j,k}}{2\Delta} \\ (D_y \psi)_{i,j,k} &= \frac{\psi_{i,j+1,k}-\psi_{i,j-1,k}}{2\Delta} \\ (D_z \psi)_{i,j,k} &= \frac{\psi_{i,j,k+1}-\psi_{i,j,k-1}}{2\Delta} \\ \end{align*} \end{frame} \begin{frame} \frametitle{Runge-Kutta time integration} Simplest Runge-Kutta method: Newton integration (1st order accurate) \begin{align*} \frac{y_{n+1}-y_n}{\Delta t} &= K_y(t_n,y_n) & \to & &y_{n+1} = y_n + K_y(t_n, y_n)\Delta t \end{align*} We will use a 2nd order accurate Runge-Kutta integration scheme (also known as a \emph{Heun method}) with two intermediate steps: \begin{align*} &k_1 = K_y(t_n, y_n) \\ &\tilde{y}_n = y_n + k_1\Delta t, \\ &k_2 = K_y(t_n + \Delta t, \tilde{y}_n) \\ &y_{n+1} = y_n + \left(\frac{1}{2} k_1 + \frac{1}{2} k_2\right)\Delta t \\ \end{align*} \end{frame} \begin{frame} \frametitle{Discrete boundary conditions} We place the grid points in such a way that the boundary of the domain $\partial\mathcal{D}$ lies in between the two last grid points: \begin{columns} \column{.7\textwidth} \begin{center} \includegraphics[width=7cm]{badwave_bcs}\\ \end{center} \column{.3\textwidth} \begin{align*} \psi|_{\partial\mathcal{D}} &= 0 \\ \vec{p}_\parallel|_{\partial\mathcal{D}} &= 0 \\ \vec{p}_\perp|_{\partial\mathcal{D}_{i+}} &= \vec{p}_\perp|_{\partial\mathcal{D}_{i-}} \\ &\Downarrow \\ \psi_{0,j,k}&=-\psi_{1,j,k}, \\ \vec{p}_{y,0,j,k} &=-\vec{p}_{y,1,j,k}, \\ \vec{p}_{z,0,j,k} &=-\vec{p}_{z,1,j,k}, \\ \vec{p}_{x,0,j,k} &= \vec{p}_{x,1,j,k}, \end{align*} \emph{... etc.} \end{columns} \end{frame} \subsection{Standalone Code} \begin{frame}[containsverbatim] \frametitle{Standalone code} \begin{columns} \column{.7\textwidth} {\small \begin{Verbatim}[commandchars=@\[\]] @PYaJ[int] @PYaL[main](@PYaJ[int] argc,@PYaJ[char] @PYbe[*]@PYbe[*]argv) { @PYaJ[FILE] @PYbe[*]fp @PYbe[=] fopen(@PYaB["]@PYaB[psi.d.asc]@PYaB["],@PYaB["]@PYaB[w+]@PYaB["]); BadWave_Init(); BadWave_ApplyBounds(); BadWave_print(fp); @PYaE[// evolve 100 steps] @PYay[for](@PYaJ[int] i@PYbe[=]@PYag[0];i@PYbe[<]@PYag[100];i@PYbe[+]@PYbe[+]) { iter @PYbe[=] i; BadWave_Evolve(); BadWave_ApplyBounds(); BadWave_Evolve2(); BadWave_TmpApplyBounds(); BadWave_print(fp); } fclose(fp); @PYay[return] @PYag[0]; } \end{Verbatim} }% \column{.3\textwidth} \includegraphics[width=.7\textwidth]{schedule_tree}\\ \end{columns} \end{frame} \begin{frame}[containsverbatim] \frametitle{Standalone code: parameters and definitions} {\small \begin{Verbatim}[commandchars=@\[\]] @PYaE[// Parameters: ] @PYaJ[bool] verbose @PYbe[=] @PYaX[true]; @PYaE[// enable verbose output] @PYaJ[double] wave_speed @PYbe[=] @PYaw[1.0]; @PYaE[// wave speed factor] @PYay[const] @PYaJ[int] isiz @PYbe[=] @PYag[32], jsiz @PYbe[=] @PYag[32], ksiz @PYbe[=] @PYag[32]; @PYaE[//grid size] @PYaE[// 1D array to hold the grid and a function to find 3D indices] @PYay[typedef] @PYaJ[double] gridfunc@PYZlb[]isiz@PYbe[*]jsiz@PYbe[*]ksiz@PYZrb[]; @PYap[inline] @PYaJ[int] @PYaL[INDEX3D](@PYaJ[int] i,@PYaJ[int] j,@PYaJ[int] k) { @PYay[return] (i@PYbe[+]j@PYbe[*]isiz)@PYbe[*]ksiz@PYbe[+]k; } @PYaE[// Grid functions:] @PYaJ[gridfunc] psi,px,py,pz; @PYaE[// psi and p] @PYaJ[gridfunc] tpsi,tpx,tpy,tpz; @PYaE[// temporary variables] @PYaJ[gridfunc] kpsi,kpx,kpy,kpz; @PYaE[// the kernels] @PYaJ[gridfunc] k2psi,k2px,k2py,k2pz; @PYaE[// Other global variables: iteration, grid spacing and time step] @PYaJ[int] iter@PYbe[=]@PYag[0]; @PYaJ[double] dx, dy, dz, dt; \end{Verbatim} } %small \end{frame} \subsection{Thorn BadWave} \begin{frame}[containsverbatim] \frametitle{Writing thorn BadWave: \tt{param.ccl}} \begin{Verbatim}[commandchars=@\[\]] @PYaN[#]@PYaN[ Parameter definitions for thorn BadWave] @PYah[private:] @PYaJ[CCTK_REAL] wave_speed @PYaB["]@PYaB[characteristic speed at boundary]@PYaB["] { @PYaB["]@PYaB[0.1:*]@PYaB["] @PYbe[:]@PYbe[:] @PYaB["]@PYaB[wave speed]@PYaB["] } @PYaw[1.] @PYah[private:] @PYaJ[CCTK_BOOLEAN] verbose @PYaB["]@PYaB[whether to print stuff]@PYaB["] { @PYbe[:]@PYbe[:] @PYaB["]@PYaB[verbose flag]@PYaB["] } @PYag[0] \end{Verbatim} \end{frame} \begin{frame}[containsverbatim] \frametitle{Writing thorn BadWave: \tt{interface.ccl}} \begin{Verbatim}[commandchars=@\[\]] @PYaN[#]@PYaN[ Interface definition for thorn BadWave] @PYah[implements:] BadWaveTutorial @PYah[inherits:] grid @PYaJ[CCTK_REAL] wave_vars TYPE@PYbe[=]gf { psi px, py, pz } @PYaB["]@PYaB[Basic components of wave equation code]@PYaB["] @PYaJ[CCTK_REAL] tmp_wave_vars TYPE@PYbe[=]gf { tpsi, tpx, tpy, tpz } <...> @PYaJ[CCTK_REAL] kernels TYPE@PYbe[=]gf { kpsi kpx, kpy, kpz, k2psi, k2px, k2py, k2pz } <...> \end{Verbatim} \end{frame} \begin{frame}[containsverbatim] \frametitle{Writing thorn BadWave: \tt{schedule.ccl}} {\small \begin{Verbatim}[commandchars=@\[\]] @PYaN[#]@PYaN[ Schedule definitions for thorn BadWave] @PYah[storage:] wave_vars, tmp_wave_vars, kernels @PYay[schedule] BadWave_Init @PYay[at] @PYaL[INITIAL] { @PYah[LANG:] C } @PYaB["]@PYaB[Startup and initialize]@PYaB["] @PYay[schedule] BadWave_ApplyBounds @PYay[at] @PYaL[POSTSTEP] { @PYah[LANG:] C @PYah[SYNC:] wave_vars } @PYaB["]@PYaB[Apply boundary conditions]@PYaB["] @PYay[schedule] BadWave_Evolve @PYay[at] @PYaL[EVOL] <...> @PYay[schedule] BadWave_TmpApplyBounds @PYay[at] @PYaL[EVOL] @PYay[after] BadWave_Evolve <...> @PYay[schedule] BadWave_Evolve2 @PYay[at] @PYaL[EVOL] @PYay[after] BadWave_TmpApplyBounds <...> \end{Verbatim} } %small \end{frame} \begin{frame}[containsverbatim] \frametitle{Adapting the source code for Cactus} Modifications to standalone code: \begin{itemize} \item include Cactus headers: {\small \begin{Verbatim}[commandchars=@\[\]] @PYaN[#]@PYaN[include "cctk.h"] @PYaN[#]@PYaN[include "cctk_Arguments.h"] @PYaN[#]@PYaN[include "cctk_Parameters.h"] \end{Verbatim} } %small \item declare all scheduled functions with external C linkage: {\small \begin{Verbatim}[commandchars=@\[\]] @PYay[extern] @PYaB["]@PYaB[C]@PYaB["] @PYaJ[void] BadWave_Init(CCTK_ARGUMENTS); @PYay[extern] @PYaB["]@PYaB[C]@PYaB["] @PYaJ[void] BadWave_ApplyBounds(CCTK_ARGUMENTS); @PYay[extern] @PYaB["]@PYaB[C]@PYaB["] @PYaJ[void] BadWave_TmpApplyBounds(CCTK_ARGUMENTS); @PYay[extern] @PYaB["]@PYaB[C]@PYaB["] @PYaJ[void] BadWave_Evolve(CCTK_ARGUMENTS); @PYay[extern] @PYaB["]@PYaB[C]@PYaB["] @PYaJ[void] BadWave_Evolve2(CCTK_ARGUMENTS); \end{Verbatim} } %small \item add the {\tt CCTK\_ARGUMENTS} macro to scheduled functions: {\small \begin{Verbatim}[commandchars=@\[\]] @PYaJ[void] @PYaL[BadWave_Init](CCTK_ARGUMENTS) { DECLARE_CCTK_ARGUMENTS; ... } \end{Verbatim} } %small this tells functions which part of the grid they will be working on. \end{itemize} \end{frame} \begin{frame}[containsverbatim] \frametitle{Grid definitions in Cactus} \begin{columns} \column{.4\textwidth} \begin{center} \includegraphics[width=\textwidth]{cctk_grid_defns}\\ \end{center} \column{.6\textwidth} \begin{itemize} \item {\tt cctk\_gsh}: global grid dimensions \item {\tt cctk\_lsh}: local grid dimensions \item {\tt cctk\_delta\_space}: grid spacing \item {\tt cctk\_nghostzones}: number of ghostzones \item {\tt cctk\_bbox}: whether a boundary is an outer one \item {\tt CCTK\_GFINDEX3D(i,j,k)}: computes local 1D grid index \end{itemize} \end{columns} \end{frame} \subsection{Running BadWave} \begin{frame}[containsverbatim] \frametitle{Simple parameter file} Parfile: {\tt par/BadWavePUGH.par} \begin{itemize} \item include the driver thorn (PUGH) \item specify grid size \item specify BadWave parameters \end{itemize} \begin{Verbatim}[commandchars=@\[\]] ActiveThorns @PYbe[=] @PYaB["]@PYaB[BadWave PUGH]@PYaB["] cactus@PYbe[:]@PYbe[:]cctk_itlast @PYbe[=] @PYag[100] pugh@PYbe[:]@PYbe[:]global_nsize @PYbe[=] @PYag[32] pugh@PYbe[:]@PYbe[:]ghost_size @PYbe[=] @PYag[1] BadWavePUGH@PYbe[:]@PYbe[:]wave_speed @PYbe[=] @PYaw[2.0] \end{Verbatim} \end{frame} \begin{frame}[containsverbatim] \frametitle{More advanced parameter file} Parfile: {\tt par/BadWavePUGHv2.par} Includes new thorns: \begin{itemize} \item {\tt CoordBase}: coordinate extents \item {\tt CardGrid3D}: provides variety of grid configurations \item {\tt SymBase}: basic thorn for specifying symmetries of the domain \item {\tt IOBasic, IOUtil, IOScalar, IOASCII}: basic and advanced I/O \item {\tt Time}: provides global time and iteration variables \item {\tt PUGHSlab, PUGHReduce, LocalReduce}: required by I/O thorns \end{itemize} \end{frame} \subsection{Method of Lines} \begin{frame} \frametitle{Method of Lines} \begin{itemize} \item Method of lines is a general name for an approach of solving PDEs with time variable, in which the PDE is approximated by a system of interdependent ODEs for each grid point. \item Method of lines allows to extend time integration methods developed for ODEs to PDEs. \item Cactus thorn MoL implements several different time integration methods. \item We can use these methods if we modify our thorn: {\tt BadWaveMoL} \end{itemize} \end{frame} \begin{frame}[containsverbatim] \frametitle{Registering evolved variables with MoL} \begin{itemize} \item in Wave.cc: {\small \begin{Verbatim}[commandchars=@\[\]] @PYaJ[void] @PYaL[BadWaveMoL_Register](CCTK_ARGUMENTS) { DECLARE_CCTK_ARGUMENTS; MoLRegisterEvolved(CCTK_VarIndex(@PYaB["]@PYaB[BadWaveMoL::psi]@PYaB["]), CCTK_VarIndex(@PYaB["]@PYaB[BadWaveMoL::kpsi]@PYaB["])); ... } \end{Verbatim} } %small \item in {\tt schedule.ccl}: \begin{Verbatim}[commandchars=@\[\]] @PYah[storage:] wave_vars@PYZlb[]@PYag[3]@PYZrb[], kernels@PYZlb[]@PYag[3]@PYZrb[] schedule BadWaveMoL_Register in MoL_Register { @PYah[LANG:] C } @PYaB["]@PYaB[Register for MoL]@PYaB["] \end{Verbatim} \end{itemize} \end{frame} \begin{frame}[containsverbatim] \frametitle{Registering evolved variables with MoL} \begin{itemize} \item in {\tt interface.ccl}: {\small \begin{Verbatim}[commandchars=@\[\]] CCTK_INT FUNCTION MoLRegisterEvolved \ (CCTK_INT IN EvolvedIndex, CCTK_INT IN RHSIndex) USES FUNCTION MoLRegisterEvolved \end{Verbatim} } \item in the parfile ({\tt par/BadWaveMoL.par}): \begin{verbatim} ActiveThorns = "BadWaveMoL carpet CarpetIOBasic CarpetIOASCII CarpetIOScalar CarpetLib LoopControl CoordBase IOUtil SymBase Time CarpetReduce CartGrid3D MoL" MoL::ODE_Method = "RK4" MoL::MoL_Intermediate_Steps = 4 MoL::MoL_Num_Scratch_Levels = 1 \end{verbatim} \end{itemize} \end{frame} \section{Mesh Refinement} \subsection{Fixed mesh refinement} \begin{frame}[containsverbatim] \frametitle{Berger-Oliger mesh refinement} \begin{columns} \column{0.5\textwidth} \begin{itemize} \item "Box-in-box" refinement: \begin{center} \includegraphics[width=0.6\textwidth]{cactus_fmr} \end{center} \item Time update: \begin{center} \includegraphics[width=0.6\textwidth]{cactus_fmr_update} \end{center} \end{itemize} \column{0.5\textwidth} {\footnotesize \begin{minipage}[t]{0.8\textwidth} \include{CBO} \end{minipage} } %footnotesize \end{columns} \end{frame} \begin{frame}[containsverbatim] \frametitle{Fixed mesh refinement} Parfile: {\tt par/BadWaveFMR.par} {\small \begin{Verbatim}[commandchars=@\[\]] ActiveThorns @PYbe[=] @PYaB["]@PYaB[CarpetRegrid2 Dissipation] @PYaB[SpaceMask SphericalSurface]@PYaB["] CarpetRegrid2@PYbe[:]@PYbe[:]num_levels_1 @PYbe[=] @PYag[2] Carpet@PYbe[:]@PYbe[:]max_refinement_levels @PYbe[=] @PYag[2] CarpetRegrid2@PYbe[:]@PYbe[:]num_centres @PYbe[=] @PYag[1] CarpetRegrid2@PYbe[:]@PYbe[:]Position_x_1 @PYbe[=] @PYaw[0.5] CarpetRegrid2@PYbe[:]@PYbe[:]Position_y_1 @PYbe[=] @PYaw[0.5] CarpetRegrid2@PYbe[:]@PYbe[:]Position_z_1 @PYbe[=] @PYaw[0.5] CarpetRegrid2@PYbe[:]@PYbe[:]radius_1@PYZlb[]@PYag[1]@PYZrb[] @PYbe[=] @PYaw[0.1] @PYaN[#]@PYaN[CarpetRegrid2::radius_1@PYZlb[]2@PYZrb[] = 0.1] CarpetRegrid2@PYbe[:]@PYbe[:]verbose @PYbe[=] @PYaB["]@PYaB[yes]@PYaB["] Carpet@PYbe[:]@PYbe[:]init_fill_timelevels@PYbe[=]@PYaB["]@PYaB[yes]@PYaB["] Dissipation@PYbe[:]@PYbe[:]vars @PYbe[=] @PYaB["]@PYaB[BadWaveMoL::psi]@PYaB["] \end{Verbatim} } \end{frame} \subsection{Adaptive mesh refinement} \begin{frame}[containsverbatim] \frametitle{Adaptive mesh refinement} Modifications to the code: {\footnotesize \begin{Verbatim}[commandchars=@\[\]] @PYaJ[void] @PYaL[BadWaveAMR_BoxMover](CCTK_ARGUMENTS) { DECLARE_CCTK_ARGUMENTS; @PYaJ[int] max_refinement_levels @PYbe[=] @PYag[30]; radius@PYZlb[]max_refinement_levels@PYbe[*]@PYag[0]@PYbe[+]@PYag[1]@PYZrb[]@PYbe[=]@PYaw[.05]; @PYaE[// Make the box wiggle] position_x@PYZlb[]@PYag[0]@PYZrb[] @PYbe[=] position_y@PYZlb[]@PYag[0]@PYZrb[] @PYbe[=] position_z@PYZlb[]@PYag[0]@PYZrb[] @PYbe[=] @PYaw[0.7]@PYbe[+]@PYaw[0.05]@PYbe[*]sin(cctk_time); active@PYZlb[]@PYag[0]@PYZrb[]@PYbe[=]@PYag[1]; num_levels@PYZlb[]@PYag[0]@PYZrb[]@PYbe[=]@PYag[2]; @PYaE[// two levels: 0=base grid, 1=first refined grid] @PYaE[// Turn on the next box] active@PYZlb[]@PYag[1]@PYZrb[] @PYbe[=] @PYag[1]; radius@PYZlb[]max_refinement_levels@PYbe[*]@PYag[1]@PYbe[+]@PYag[1]@PYZrb[]@PYbe[=]@PYaw[.05]; position_x@PYZlb[]@PYag[1]@PYZrb[]@PYbe[=]position_y@PYZlb[]@PYag[1]@PYZrb[]@PYbe[=]position_z@PYZlb[]@PYag[1]@PYZrb[]@PYbe[=]@PYaw[.3]; num_levels@PYZlb[]@PYag[1]@PYZrb[]@PYbe[=]@PYag[2]; } \end{Verbatim} } \end{frame} \begin{frame}[containsverbatim] \frametitle{Adaptive mesh refinement} \begin{itemize} \item Modifications to the {\tt schedule.ccl}: \begin{Verbatim}[commandchars=@\[\]] schedule BadWaveAMR_BoxMover at preregrid { @PYah[LANG:] C } @PYaB["]@PYaB[Jiggle the box]@PYaB["] \end{Verbatim} \item Modifications to the {\tt interface.ccl}: \begin{verbatim} inherits: grid, CarpetRegrid2 \end{verbatim} \end{itemize} \end{frame} \begin{frame}[containsverbatim] \frametitle{Adaptive mesh refinement} Modifications in the parfile ({\tt pars/BadWaveAMR.par}): {\small \begin{Verbatim}[commandchars=@\[\]] CarpetRegrid2@PYbe[:]@PYbe[:]regrid_every @PYbe[=] @PYag[2] Carpet@PYbe[:]@PYbe[:]max_refinement_levels @PYbe[=] @PYag[3] CarpetRegrid2@PYbe[:]@PYbe[:]num_centres @PYbe[=] @PYag[2] CarpetRegrid2@PYbe[:]@PYbe[:]num_levels_1 @PYbe[=] @PYag[3] CarpetRegrid2@PYbe[:]@PYbe[:]Position_x_1 @PYbe[=] @PYaw[0.3] CarpetRegrid2@PYbe[:]@PYbe[:]Position_y_1 @PYbe[=] @PYaw[0.3] CarpetRegrid2@PYbe[:]@PYbe[:]Position_z_1 @PYbe[=] @PYaw[0.3] CarpetRegrid2@PYbe[:]@PYbe[:]radius_1@PYZlb[]@PYag[1]@PYZrb[] @PYbe[=] @PYaw[0.2] CarpetRegrid2@PYbe[:]@PYbe[:]radius_1@PYZlb[]@PYag[2]@PYZrb[] @PYbe[=] @PYaw[0.1] CarpetRegrid2@PYbe[:]@PYbe[:]num_levels_2 @PYbe[=] @PYag[2] CarpetRegrid2@PYbe[:]@PYbe[:]Position_x_2 @PYbe[=] @PYaw[0.7] CarpetRegrid2@PYbe[:]@PYbe[:]Position_y_2 @PYbe[=] @PYaw[0.7] CarpetRegrid2@PYbe[:]@PYbe[:]Position_z_2 @PYbe[=] @PYaw[0.7] CarpetRegrid2@PYbe[:]@PYbe[:]radius_2@PYZlb[]@PYag[1]@PYZrb[] @PYbe[=] @PYaw[0.05] \end{Verbatim} } \end{frame} \begin{frame}[containsverbatim] \frametitle{Adaptive mesh refinement} Modifications in the parfile ({\tt pars/BadWaveAMR.par}): {\footnotesize \begin{Verbatim}[commandchars=@\[\]] ActiveThorns @PYbe[=] @PYaB["]@PYaB[IOJPeg CarpetSlab CarpetInterp ] @PYaB[CarpetInterp2 AEILocalInterp]@PYaB["] IOJpeg@PYbe[:]@PYbe[:]out_every @PYbe[=] @PYag[1] IOJpeg@PYbe[:]@PYbe[:]out_vars @PYbe[=] @PYaB["]@PYaB[BadWaveAMR::psi]@PYaB["] IOJPeg@PYbe[:]@PYbe[:]gridpoints @PYbe[=] interpolate IOJpeg@PYbe[:]@PYbe[:]array2d_x0 @PYbe[=] @PYaw[0.0] IOJpeg@PYbe[:]@PYbe[:]array2d_y0 @PYbe[=] @PYaw[0.0] IOJpeg@PYbe[:]@PYbe[:]array2d_z0 @PYbe[=] @PYaw[0.5] IOJpeg@PYbe[:]@PYbe[:]array2d_npoints_i @PYbe[=] @PYag[101] IOJpeg@PYbe[:]@PYbe[:]array2d_dx_i @PYbe[=] @PYaw[0.01] IOJpeg@PYbe[:]@PYbe[:]array2d_dy_i @PYbe[=] @PYag[0] IOJpeg@PYbe[:]@PYbe[:]array2d_dz_i @PYbe[=] @PYag[0] IOJpeg@PYbe[:]@PYbe[:]array2d_npoints_j @PYbe[=] @PYag[101] IOJpeg@PYbe[:]@PYbe[:]array2d_dx_j @PYbe[=] @PYag[0] IOJpeg@PYbe[:]@PYbe[:]array2d_dy_j @PYbe[=] @PYaw[0.01] IOJpeg@PYbe[:]@PYbe[:]array2d_dz_j @PYbe[=] @PYag[0] \end{Verbatim} } \end{frame}