fix issue 82

Source/CAMR.H

@@ -157,17 +157,24 @@ public:
                                       amrex::Real time,
                                       amrex::Real dt);
 
+  //
+  // Build the old-time source terms on the valid region plus ng ghost cells.
+  // S must be a state with at least ng ghost cells filled (i.e. Sborder, not
+  // the state data itself) since the sources are evaluated cell-by-cell from it.
+  //
   void construct_old_source (
     int src,
+    const amrex::MultiFab& S,
     amrex::Real time,
-    amrex::Real dt);
+    amrex::Real dt,
+    int ng);
 
   void construct_new_source (
     int src,
     amrex::Real time,
     amrex::Real dt);
 
-  void construct_old_ext_source(amrex::Real time, amrex::Real dt);
+  void construct_old_ext_source(const amrex::MultiFab& S, amrex::Real time, amrex::Real dt, int ng);
   void construct_new_ext_source(amrex::Real time, amrex::Real dt);
 
   static void fill_ext_source (
@@ -178,7 +185,7 @@ public:
     amrex::MultiFab& ext_src,
     int ng);
 
-  void construct_old_grav_source (amrex::Real time, amrex::Real dt);
+  void construct_old_grav_source (const amrex::MultiFab& S, amrex::Real time, amrex::Real dt, int ng);
   void construct_new_grav_source (amrex::Real time, amrex::Real dt);
 
   static void fill_grav_source (const amrex::MultiFab& S, amrex::MultiFab& grav_src, int ng);

Source/CAMR_advance.cpp

@@ -111,9 +111,15 @@ CAMR::CAMR_advance (Real time,
     // Initialize the new-time data.
     MultiFab::Copy(S_new, Sborder, 0, 0, NVAR, S_new.nGrow());
 
-    // Build sources at t_old, then add them to S_new
+    // Build sources at t_old, then add them to S_new.
+    //
+    // The Godunov hydro reads these sources (via sources_for_hydro --> srcQ) in
+    // the ghost cells as well, so we must build them over the full numGrow()
+    // footprint, using Sborder since the state itself carries no ghost cells.
+    // The MOL hydro never builds srcQ, so there we only need the valid region.
+    int ng_src = (do_mol) ? 0 : numGrow();
     for (int n = 0; n < src_list.size(); ++n) {
-        construct_old_source(src_list[n], time, dt);
+        construct_old_source(src_list[n], Sborder, time, dt, ng_src);
         MultiFab::Saxpy(S_new, dt, *old_sources[src_list[n]], 0, 0, NVAR, 0);
     }
 

Source/CAMR_sources.cpp

@@ -4,19 +4,22 @@
 void
 CAMR::construct_old_source(
   int src,
+  const amrex::MultiFab& S,
   amrex::Real time,
-  amrex::Real dt)
+  amrex::Real dt,
+  int ng)
 {
   AMREX_ASSERT(src >= 0 && src < num_src);
+  AMREX_ASSERT(S.nGrow() >= ng);
 
   switch (src) {
 
   case ext_src:
-    construct_old_ext_source(time, dt);
+    construct_old_ext_source(S, time, dt, ng);
     break;
 
   case grav_src:
-    construct_old_grav_source(time, dt);
+    construct_old_grav_source(S, time, dt, ng);
     break;
   } // end switch
 }

Source/GravitySrc.cpp

@@ -1,17 +1,23 @@
 #include "CAMR.H"
 
 void
-CAMR::construct_old_grav_source (amrex::Real /*time*/, amrex::Real /*dt*/)
+CAMR::construct_old_grav_source (const amrex::MultiFab& S,
+                                 amrex::Real /*time*/, amrex::Real /*dt*/,
+                                 int ng)
 {
   old_sources[grav_src]->setVal(0.0);
 
   if (!add_grav_src) {
     return;
   }
 
-  const amrex::MultiFab& S_old = get_old_data(State_Type);
-  int ng = S_old.nGrow();
-  fill_grav_source(S_old, *old_sources[grav_src], ng);
+  // Note that S must be a state with at least ng ghost cells filled (i.e.
+  // Sborder rather than the state data, which carries no ghost cells) because
+  // the hydro reads the source terms in the ghost cells as well, and neither
+  // FillBoundary here nor in construct_hydro_source can fill the ghost cells
+  // at a coarse-fine boundary.
+  AMREX_ASSERT(S.nGrow() >= ng);
+  fill_grav_source(S, *old_sources[grav_src], ng);
 
   old_sources[grav_src]->FillBoundary(geom.periodicity());
 }

Source/Hydro/CAMR_construct_hydro_source.cpp

@@ -25,8 +25,13 @@ CAMR::construct_hydro_source (const MultiFab& S,
 
     AMREX_ASSERT(S.nGrow() == numGrow());
 
-    // Fill the source terms to go into the hydro with only the old-time sources
-    int ng = 0;
+    // Fill the source terms to go into the hydro with only the old-time sources.
+    //
+    // The Godunov trace reads srcQ in the ghost cells, so we must include the
+    // ghost cells of the old-time sources here -- the FillBoundary below only
+    // copies valid data from the same level, so it cannot fill the ghost cells
+    // at a coarse-fine boundary.  (The MOL path never builds srcQ.)
+    int ng = (do_mol) ? 0 : numGrow();
 
     for (int n = 0; n < src_list.size(); ++n) {
         MultiFab::Saxpy(sources_for_hydro, 1.0, *old_sources[src_list[n]], 0, 0, NVAR, ng);

Source/Utils/External.cpp

@@ -2,19 +2,20 @@
 #include "IndexDefines.H"
 
 void
-CAMR::construct_old_ext_source(amrex::Real time, amrex::Real dt)
+CAMR::construct_old_ext_source(const amrex::MultiFab& S,
+                               amrex::Real time, amrex::Real dt, int ng)
 {
-  const amrex::MultiFab& S_old = get_old_data(State_Type);
-
-  int ng = 0; // None filled
-
   old_sources[ext_src]->setVal(0.0);
 
   if (!add_ext_src) {
     return;
   }
 
-  fill_ext_source(time, dt, S_old, S_old, *old_sources[ext_src], ng);
+  // Note that S must be a state with at least ng ghost cells filled (i.e.
+  // Sborder rather than the state data, which carries no ghost cells) because
+  // the hydro reads the source terms in the ghost cells as well.
+  AMREX_ASSERT(S.nGrow() >= ng);
+  fill_ext_source(time, dt, S, S, *old_sources[ext_src], ng);
 
   old_sources[ext_src]->FillBoundary(geom.periodicity());
 }