Merge branch 'develop' into tupek/contact_adjoint

.gitlab/build_toss4.yml

@@ -42,7 +42,7 @@ toss4-clang_14_0_6-src:
     COMPILER: "[email protected]"
     HOST_CONFIG: "ruby-toss_4_x86_64_ib-${COMPILER}.cmake"
     EXTRA_CMAKE_OPTIONS: "-DENABLE_BENCHMARKS=ON"
-    DO_INTEGRATION_TESTS: "yes"
+    #DO_INTEGRATION_TESTS: "yes"
     ALLOC_NODES: "2"
     ALLOC_TIME: "30"
     ALLOC_DEADLINE: "60"

cmake/SeracBasics.cmake

@@ -43,7 +43,9 @@ endif()
 
 option(SERAC_ENABLE_PROFILING "Enable profiling functionality" OFF)
 
-cmake_dependent_option(SERAC_ENABLE_BENCHMARKS "Enable benchmark executables" ON "ENABLE_BENCHMARKS" OFF)
+if (ENABLE_BENCHMARKS)
+    set(SERAC_ENABLE_BENCHMARKS ON)
+endif()
 
 # User turned on benchmarking but explicitly turned off profiling. Error out.
 if ((ENABLE_BENCHMARKS OR SERAC_ENABLE_BENCHMARKS) AND NOT SERAC_ENABLE_PROFILING)

examples/buckling/cylinder.cpp

@@ -124,9 +124,8 @@ int main(int argc, char* argv[])
 
   // Create and refine mesh
   std::string filename = SERAC_REPO_DIR "/data/meshes/hollow-cylinder.mesh";
-  auto        mesh     = serac::buildMeshFromFile(filename);
-  auto        pmesh    = mesh::refineAndDistribute(std::move(mesh), serial_refinement, parallel_refinement);
-  serac::StateManager::setMesh(std::move(pmesh), mesh_tag);
+  auto  mesh  = mesh::refineAndDistribute(serac::buildMeshFromFile(filename), serial_refinement, parallel_refinement);
+  auto& pmesh = serac::StateManager::setMesh(std::move(mesh), mesh_tag);
 
   // Surface attributes for boundary conditions
   std::set<int> xneg{2};
@@ -160,8 +159,8 @@ int main(int argc, char* argv[])
   auto                        lambda = 1.0;
   auto                        G      = 0.1;
   solid_mechanics::NeoHookean mat{.density = 1.0, .K = (3 * lambda + 2 * G) / 3, .G = G};
-
-  solid_solver->setMaterial(mat);
+  Domain                      whole_mesh = EntireDomain(pmesh);
+  solid_solver->setMaterial(mat, whole_mesh);
 
   // Set up essential boundary conditions
   // Bottom of cylinder is fixed

examples/contact/beam_bending.cpp

@@ -35,8 +35,8 @@ int main(int argc, char* argv[])
   // Construct the appropriate dimension mesh and give it to the data store
   std::string filename = SERAC_REPO_DIR "/data/meshes/beam-hex-with-contact-block.mesh";
 
-  auto mesh = serac::mesh::refineAndDistribute(serac::buildMeshFromFile(filename), 2, 0);
-  serac::StateManager::setMesh(std::move(mesh), "beam_mesh");
+  auto  mesh  = serac::mesh::refineAndDistribute(serac::buildMeshFromFile(filename), 2, 0);
+  auto& pmesh = serac::StateManager::setMesh(std::move(mesh), "beam_mesh");
 
   serac::LinearSolverOptions linear_options{.linear_solver = serac::LinearSolver::Strumpack, .print_level = 1};
 #ifndef MFEM_USE_STRUMPACK
@@ -78,7 +78,8 @@ int main(int argc, char* argv[])
   solid_solver.setParameter(1, G_field);
 
   serac::solid_mechanics::ParameterizedNeoHookeanSolid mat{1.0, 0.0, 0.0};
-  solid_solver.setMaterial(serac::DependsOn<0, 1>{}, mat);
+  serac::Domain                                        whole_mesh = serac::EntireDomain(pmesh);
+  solid_solver.setMaterial(serac::DependsOn<0, 1>{}, mat, whole_mesh);
 
   // Pass the BC information to the solver object
   solid_solver.setDisplacementBCs({1}, [](const mfem::Vector&, mfem::Vector& u) {
@@ -117,4 +118,4 @@ int main(int argc, char* argv[])
   serac::exitGracefully();
 
   return 0;
-}
+}

examples/contact/ironing.cpp

@@ -35,10 +35,11 @@ int main(int argc, char* argv[])
   // Construct the appropriate dimension mesh and give it to the data store
   std::string filename = SERAC_REPO_DIR "/data/meshes/ironing.mesh";
 
-  auto mesh = serac::mesh::refineAndDistribute(serac::buildMeshFromFile(filename), 2, 0);
-  serac::StateManager::setMesh(std::move(mesh), "ironing_mesh");
+  auto  mesh  = serac::mesh::refineAndDistribute(serac::buildMeshFromFile(filename), 2, 0);
+  auto& pmesh = serac::StateManager::setMesh(std::move(mesh), "ironing_mesh");
 
   serac::LinearSolverOptions linear_options{.linear_solver = serac::LinearSolver::Strumpack, .print_level = 1};
+
 #ifndef MFEM_USE_STRUMPACK
   SLIC_INFO_ROOT("Contact requires MFEM built with strumpack.");
   return 1;
@@ -78,7 +79,8 @@ int main(int argc, char* argv[])
   solid_solver.setParameter(1, G_field);
 
   serac::solid_mechanics::ParameterizedNeoHookeanSolid mat{1.0, 0.0, 0.0};
-  solid_solver.setMaterial(serac::DependsOn<0, 1>{}, mat);
+  serac::Domain                                        whole_mesh = serac::EntireDomain(pmesh);
+  solid_solver.setMaterial(serac::DependsOn<0, 1>{}, mat, whole_mesh);
 
   // Pass the BC information to the solver object
   solid_solver.setDisplacementBCs({5}, [](const mfem::Vector&, mfem::Vector& u) {

examples/contact/sphere.cpp

@@ -51,8 +51,8 @@ int main(int argc, char* argv[])
   cube_mesh.SetCurvature(p);
 
   std::vector<mfem::Mesh*> mesh_ptrs{&ball_mesh, &cube_mesh};
-  auto mesh = serac::mesh::refineAndDistribute(mfem::Mesh(mesh_ptrs.data(), static_cast<int>(mesh_ptrs.size())), 0, 0);
-  serac::StateManager::setMesh(std::move(mesh), "sphere_mesh");
+  auto  mesh = serac::mesh::refineAndDistribute(mfem::Mesh(mesh_ptrs.data(), static_cast<int>(mesh_ptrs.size())), 0, 0);
+  auto& pmesh = serac::StateManager::setMesh(std::move(mesh), "sphere_mesh");
 
   serac::LinearSolverOptions linear_options{.linear_solver = serac::LinearSolver::Strumpack, .print_level = 1};
 #ifndef MFEM_USE_STRUMPACK
@@ -75,7 +75,8 @@ int main(int argc, char* argv[])
       nonlinear_options, linear_options, serac::solid_mechanics::default_quasistatic_options, name, "sphere_mesh");
 
   serac::solid_mechanics::NeoHookean mat{1.0, 10.0, 0.25};
-  solid_solver.setMaterial(mat);
+  serac::Domain                      whole_mesh = serac::EntireDomain(pmesh);
+  solid_solver.setMaterial(mat, whole_mesh);
 
   // Pass the BC information to the solver object
   solid_solver.setDisplacementBCs({3}, [](const mfem::Vector&, mfem::Vector& u) {
@@ -122,4 +123,4 @@ int main(int argc, char* argv[])
   serac::exitGracefully();
 
   return 0;
-}
+}

examples/contact/twist.cpp

@@ -37,8 +37,8 @@ int main(int argc, char* argv[])
   // Construct the appropriate dimension mesh and give it to the data store
   std::string filename = SERAC_REPO_DIR "/data/meshes/twohex_for_contact.mesh";
 
-  auto mesh = serac::mesh::refineAndDistribute(serac::buildMeshFromFile(filename), 3, 0);
-  serac::StateManager::setMesh(std::move(mesh), "twist_mesh");
+  auto  mesh  = serac::mesh::refineAndDistribute(serac::buildMeshFromFile(filename), 3, 0);
+  auto& pmesh = serac::StateManager::setMesh(std::move(mesh), "twist_mesh");
 
   serac::LinearSolverOptions linear_options{.linear_solver = serac::LinearSolver::Strumpack, .print_level = 1};
 #ifndef MFEM_USE_STRUMPACK
@@ -61,7 +61,8 @@ int main(int argc, char* argv[])
       nonlinear_options, linear_options, serac::solid_mechanics::default_quasistatic_options, name, "twist_mesh");
 
   serac::solid_mechanics::NeoHookean mat{1.0, 10.0, 10.0};
-  solid_solver.setMaterial(mat);
+  serac::Domain                      whole_mesh = serac::EntireDomain(pmesh);
+  solid_solver.setMaterial(mat, whole_mesh);
 
   // Pass the BC information to the solver object
   solid_solver.setDisplacementBCs({3}, [](const mfem::Vector&, mfem::Vector& u) {
@@ -108,4 +109,4 @@ int main(int argc, char* argv[])
   serac::exitGracefully();
 
   return 0;
-}
+}

examples/simple_conduction/without_input_file.cpp

@@ -38,7 +38,7 @@ int main(int argc, char* argv[])
 
   std::string mesh_tag{"mesh"};
 
-  serac::StateManager::setMesh(std::move(mesh), mesh_tag);
+  auto& pmesh = serac::StateManager::setMesh(std::move(mesh), mesh_tag);
   // _create_mesh_end
 
   // _create_module_start
@@ -54,7 +54,9 @@ int main(int argc, char* argv[])
   // _conductivity_start
   constexpr double                               kappa = 0.5;
   serac::heat_transfer::LinearIsotropicConductor mat(1.0, 1.0, kappa);
-  heat_transfer.setMaterial(mat);
+
+  serac::Domain whole_domain = serac::EntireDomain(pmesh);
+  heat_transfer.setMaterial(mat, whole_domain);
 
   // _conductivity_end
   // _bc_start

scripts/llnl/common_build_functions.py

@@ -638,8 +638,7 @@ def get_build_dir(prefix, host_config):
 
 
 def get_repo_dir():
-    script_dir = os.path.dirname(os.path.realpath(__file__))
-    return os.path.abspath(pjoin(script_dir, "../.."))
+    return os.path.abspath(pjoin(get_script_dir(), "../.."))
 
 
 def get_build_and_test_root(prefix, timestamp):
@@ -685,7 +684,7 @@ def get_shared_spot_dir():
 
 
 def get_uberenv_path():
-    return pjoin(get_script_dir(), "../uberenv/uberenv.py")
+    return pjoin(get_repo_dir(), "scripts/uberenv/uberenv.py")
 
 
 def on_rz():
@@ -703,7 +702,7 @@ def get_script_dir():
 def get_project_name():
     global _project_name
     if not _project_name:
-        uberenv_config_path = os.path.abspath(os.path.join(get_script_dir(), "../../.uberenv_config.json"))
+        uberenv_config_path = pjoin(get_repo_dir(), ".uberenv_config.json")
         _project_name = "UNKNOWN_PROJECT"
         if os.path.exists(uberenv_config_path):
             with open(uberenv_config_path) as json_file:

scripts/llnl/run_benchmarks.py

@@ -89,6 +89,12 @@ def main():
     for cali_file in cali_files:
         if os.path.exists(cali_file):
             shutil.copy2(cali_file, spot_dir)
+            # Update group and permissions of new caliper file
+            cali_file = os.path.join(spot_dir, os.path.basename(cali_file))
+            group_info = grp.getgrnam("smithdev")
+            os.chown(cali_file, -1, group_info.gr_gid)
+            os.chmod(cali_file,
+                     stat.S_IRUSR | stat.S_IWUSR | stat.S_IRGRP | stat.S_IWGRP)
 
     # Print SPOT url
     if on_rz():

src/docs/sphinx/quickstart.rst

@@ -12,7 +12,8 @@ Quickstart Guide
 Getting Serac
 -------------
 
-Serac is hosted on `GitHub <https://github.com/LLNL/serac>`_. Serac uses git submodules, so the project must be cloned recursively. Use either of the following commands to pull Serac's repository:
+Serac is hosted on `GitHub <https://github.com/LLNL/serac>`_. Serac uses git submodules, so the project must be cloned
+recursively. Use either of the following commands to pull Serac's repository:
 
 .. code-block:: bash
 
@@ -212,8 +213,7 @@ Some build options frequently used by Serac include:
 * ``ENABLE_ASAN``: Enables the Address Sanitizer for memory safety inspections, defaults to ``OFF``
 * ``SERAC_ENABLE_TESTS``: Enables Serac unit tests, defaults to ``ON``
 * ``SERAC_ENABLE_CODEVELOP``: Enables local development build of MFEM/Axom, see :ref:`codevelop-label`, defaults to ``OFF``
-* ``SERAC_USE_VDIM_ORDERING``: Sets the vector ordering to be ``byVDIM``, which is significantly faster for algebraic multigrid,
-   but may conflict with other packages if Serac is being used as a dependency, defaults to ``OFF``.
+* ``SERAC_USE_VDIM_ORDERING``: Sets the vector ordering to be ``byVDIM``, which is significantly faster for algebraic multigrid, defaults to ``ON``.
 
 Once the build has been configured, Serac can be built with the following commands:
 

src/drivers/CMakeLists.txt

@@ -4,33 +4,33 @@
 #
 # SPDX-License-Identifier: (BSD-3-Clause)
 
-blt_add_executable( NAME        serac_driver
-                    SOURCES     serac.cpp
-                    DEPENDS_ON  serac_physics serac_mesh
-                    OUTPUT_NAME serac
-                    )
-
-if (SERAC_ENABLE_TESTS)
-    set(input_files_dir ${CMAKE_CURRENT_SOURCE_DIR}/../../data/input_files/tests)
-
-    # Run basic test for the Serac driver
-    blt_add_test(NAME          serac_driver_solid
-                 COMMAND       ${CMAKE_RUNTIME_OUTPUT_DIRECTORY}/serac -o driver_solid -i ${input_files_dir}/solid/dyn_solve.lua
-                 NUM_MPI_TASKS 1 )
-
-    blt_add_test(NAME          serac_driver_heat_transfer
-                 COMMAND       ${CMAKE_RUNTIME_OUTPUT_DIRECTORY}/serac -o driver_heat_transfer -i ${input_files_dir}/heat_transfer/static_solve.lua
-                 NUM_MPI_TASKS 1 )
-
-    blt_add_test(NAME          serac_driver_help
-                 COMMAND       ${CMAKE_RUNTIME_OUTPUT_DIRECTORY}/serac --help 
-                 NUM_MPI_TASKS 1 )
-
-    blt_add_test(NAME          serac_driver_docs
-                 COMMAND       ${CMAKE_RUNTIME_OUTPUT_DIRECTORY}/serac -o docs -d -i ${input_files_dir}/solid/qs_linear.lua
-                 NUM_MPI_TASKS 1 )
-endif()
-
-install( TARGETS serac_driver
-         RUNTIME DESTINATION bin
-         )
+#blt_add_executable( NAME        serac_driver
+#                    SOURCES     serac.cpp
+#                    DEPENDS_ON  serac_physics serac_mesh
+#                    OUTPUT_NAME serac
+#                    )
+#
+#if (SERAC_ENABLE_TESTS)
+#    set(input_files_dir ${CMAKE_CURRENT_SOURCE_DIR}/../../data/input_files/tests)
+#
+#    # Run basic test for the Serac driver
+#    blt_add_test(NAME          serac_driver_solid
+#                 COMMAND       ${CMAKE_RUNTIME_OUTPUT_DIRECTORY}/serac -o driver_solid -i ${input_files_dir}/solid/dyn_solve.lua
+#                 NUM_MPI_TASKS 1 )
+#
+#    blt_add_test(NAME          serac_driver_heat_transfer
+#                 COMMAND       ${CMAKE_RUNTIME_OUTPUT_DIRECTORY}/serac -o driver_heat_transfer -i ${input_files_dir}/heat_transfer/static_solve.lua
+#                 NUM_MPI_TASKS 1 )
+#
+#    blt_add_test(NAME          serac_driver_help
+#                 COMMAND       ${CMAKE_RUNTIME_OUTPUT_DIRECTORY}/serac --help 
+#                 NUM_MPI_TASKS 1 )
+#
+#    blt_add_test(NAME          serac_driver_docs
+#                 COMMAND       ${CMAKE_RUNTIME_OUTPUT_DIRECTORY}/serac -o docs -d -i ${input_files_dir}/solid/qs_linear.lua
+#                 NUM_MPI_TASKS 1 )
+#endif()
+#
+#install( TARGETS serac_driver
+#         RUNTIME DESTINATION bin
+#         )

src/serac/infrastructure/accelerator.hpp

@@ -118,6 +118,13 @@ void zero_out(axom::Array<T, dim, space>& arr)
 {
   std::memset(arr.data(), 0, static_cast<std::size_t>(arr.size()) * sizeof(T));
 }
+
+/// @brief set the contents of an array to zero, byte-wise
+template <typename T, int dim>
+void zero_out(axom::ArrayView<T, dim, axom::MemorySpace::Host>& arr)
+{
+  std::memset(arr.data(), 0, static_cast<std::size_t>(arr.size()) * sizeof(T));
+}
 #ifdef __CUDACC__
 /// @overload
 template <typename T, int dim>

src/serac/infrastructure/debug_print.hpp

@@ -61,12 +61,6 @@ std::ostream& operator<<(std::ostream& out, DoF dof)
   return out;
 }
 
-std::ostream& operator<<(std::ostream& out, serac::SignedIndex i)
-{
-  out << "{" << i.index_ << ", " << i.sign_ << "}";
-  return out;
-}
-
 /**
  * @brief write a 2D array of values out to file, in a space-separated format
  * @tparam T the type of each value in the array

src/serac/numerics/functional/CMakeLists.txt

@@ -10,7 +10,6 @@ set(functional_depends serac_mesh ${serac_device_depends})
 set(functional_headers
     differentiate_wrt.hpp
     boundary_integral_kernels.hpp
-    dof_numbering.hpp
     element_restriction.hpp
     geometry.hpp
     geometric_factors.hpp
@@ -22,6 +21,7 @@ set(functional_headers
     function_signature.hpp
     functional_qoi.inl
     integral.hpp
+    interior_face_integral_kernels.hpp
     isotropic_tensor.hpp
     polynomials.hpp
     quadrature.hpp
@@ -30,6 +30,7 @@ set(functional_headers
     tensor.hpp
     tuple.hpp
     tuple_tensor_dual_functions.hpp
+    typedefs.hpp
     )
 
 set(functional_sources