Merge pull request #49 from CEED/gpu-opt

GPU + FOM

Author: vladotomov

.gitignore

@@ -1,4 +1,10 @@
-*.o
 remhos
+*.dat
 *.mesh
+*.o
 *.gf
+*.svg
+.DS_Store
+/build/
+/.cache/
+/.vscode/

CMakeLists.txt

@@ -0,0 +1,85 @@
+cmake_minimum_required(VERSION 3.18)
+
+set(project remhos)
+project(${project} LANGUAGES CXX)
+set(CMAKE_CXX_STANDARD 17)
+set(CMAKE_CXX_STANDARD_REQUIRED ON)
+set(CMAKE_CXX_COMPILER_LAUNCHER ccache)
+set(CMAKE_EXPORT_COMPILE_COMMANDS ON)
+
+# CXX flags *******************************************************************
+if (CMAKE_BUILD_TYPE STREQUAL "Debug")
+    add_compile_options(-fsanitize=address -O0)
+    add_link_options(-fsanitize=address)
+endif()
+
+# remove -DNDEBUG from default RelWithDebInfo flags
+set(CMAKE_CXX_FLAGS_RELWITHDEBINFO "-O2" CACHE STRING "" FORCE)
+
+# Verbosity options ***********************************************************
+set(CMAKE_VERBOSE_MAKEFILE OFF CACHE BOOL "" FORCE)
+# set(CUDA_VERBOSE_BUILD OFF CACHE BOOL "" FORCE)
+
+# Include paths ***************************************************************
+include_directories(${CMAKE_CURRENT_SOURCE_DIR})
+include_directories(${CMAKE_CURRENT_SOURCE_DIR}/../mfem)
+if (${CMAKE_SYSTEM_NAME} MATCHES "Linux")
+include_directories(/usr/include/hypre /usr/include)
+elseif(${CMAKE_SYSTEM_NAME} MATCHES "Darwin")
+include_directories(/opt/homebrew/opt/fmt/include
+                    /opt/homebrew/opt/openmpi/include
+                    /opt/homebrew/opt/metis/include
+                    /opt/homebrew/opt/hypre/include)
+add_compile_definitions(MFEM_USE_CMAKE_TESTS)
+else()
+    message(FATAL_ERROR "Unsupported system")
+endif()
+
+# Copy mesh files *************************************************************
+file(GLOB SRC_MESH_FILES LIST_DIRECTORIES false data/*.mesh)
+set(BUILD_DATA_DIR ${CMAKE_CURRENT_BINARY_DIR}/data)
+add_custom_command(OUTPUT data_is_copied
+    COMMAND ${CMAKE_COMMAND} -E make_directory ${BUILD_DATA_DIR}
+    COMMAND ${CMAKE_COMMAND} -E copy_if_different ${SRC_MESH_FILES} ${BUILD_DATA_DIR}
+    COMMAND ${CMAKE_COMMAND} -E touch data_is_copied
+    COMMENT "Copying mesh files ...")
+add_custom_target(copy_mesh_files DEPENDS data_is_copied)
+
+# Library source files ********************************************************
+add_library(Remhos STATIC remhos.cpp remhos_fct.cpp 
+                          remhos_ho.cpp remhos_lo.cpp 
+                          remhos_mono.cpp remhos_sync.cpp 
+                          remhos_tools.cpp)
+
+# Testing options *************************************************************
+enable_testing()
+function(add_remhos_test name)
+    set(file ${name}.cpp)
+    set(target ${name})
+    list(LENGTH ARGN ARGN_COUNT)
+    if(ARGN_COUNT GREATER 0) 
+        list(GET ARGN 0 extra)
+        string(APPEND name "_" ${extra})
+        string(APPEND target "_" ${extra})
+    endif()
+    message(STATUS "Adding target: ${target}")
+    add_executable(${target} ${file})
+    add_dependencies(${target} copy_mesh_files)
+    target_link_libraries(${target} Remhos -lmpi -lmfem -lhypre -lmetis -lfmt)
+if (${CMAKE_SYSTEM_NAME} MATCHES "Linux")
+    target_link_directories(${target} PUBLIC /usr/lib/x86_64-linux-gnu) 
+elseif(${CMAKE_SYSTEM_NAME} MATCHES "Darwin")
+    target_link_directories(${target} PUBLIC /opt/homebrew/opt/openmpi/lib
+                                          /opt/homebrew/opt/metis/lib
+                                          /opt/homebrew/opt/hypre/lib
+                                          /opt/homebrew/opt/fmt/lib) 
+endif()
+    target_link_directories(${target} PUBLIC ${CMAKE_CURRENT_SOURCE_DIR}/../mfem)
+    message(STATUS "Adding test: ${target}_n1")
+    add_test(NAME ${target}_n1  COMMAND mpirun -n 1 ${target} ${extra})
+    message(STATUS "Adding test: ${target}_n3")
+    add_test(NAME ${target}_n3 COMMAND mpirun -n 3 ${target})
+endfunction()
+
+add_remhos_test(remhos_main)
+add_remhos_test(remhos_tests)

README.md

@@ -260,7 +260,19 @@ round-off distance from the above reference values.
 
 ## Performance Timing and FOM
 
-To appear soon.
+Performance is tracked only for the configuration `-ho 3 -lo 5 -fct 2`.
+This configuration supports partial assembly and GPU execution.
+Remhos reports several FOMs in the terminal, based on the distinct phases of an
+advection-based remap calculation. All FOMs are reported in
+`(megaDOFs x time steps) per second`, reflecting the throughput of the calculation.
+
+- FOM RHS: construction of the right-hand side of the system.
+- FOM INV: inverting the high-order operator, which is used to obtain a
+           high-order unbounded (HO) solution.
+- FOM LO:  computation of the low-order bounded (LO) approximation of the solution.
+- FOM FCT: computation of the FCT solution, combining the LO and HO solutions to
+           obtain a bounded high-order solution.
+- **FOM**: performance metric combining all the above phases.
 
 ## Versions
 
@@ -276,7 +288,7 @@ comment in the [issue tracker](https://github.com/CEED/Remhos/issues).
 The following copyright applies to each file in the CEED software suite,
 unless otherwise stated in the file:
 
-> Copyright (c) 2017, Lawrence Livermore National Security, LLC. Produced at the
+> Copyright (c) 2026, Lawrence Livermore National Security, LLC. Produced at the
 > Lawrence Livermore National Laboratory. LLNL-CODE-734707. All Rights reserved.
 
 See files LICENSE and NOTICE for details.

makefile

@@ -64,6 +64,18 @@ TEST_MK = $(MFEM_DIR)/config/test.mk
 MFEM_DIR1 := $(MFEM_DIR)
 MFEM_DIR2 := $(realpath $(MFEM_DIR))
 
+# Use Caliper annotations
+
+ifdef CALIPER_DIR
+CALIPER_DIR = $(spack location --install-dir caliper)
+ADIAK_DIR = $(spack location --install-dir adiak)
+CALIPER_FLAGS = -I${CALIPER_DIR}/include -DUSE_CALIPER
+ADIAK_INCLUDE = -I${ADIAK_DIR}/include 
+ADIAK_LDFLAGS =  -L${ADIAK_DIR}/lib -ladiak
+CALIPER_LDFLAGS =  -L${CALIPER_DIR}/lib64 -lcaliper 
+endif
+
+
 # Use the compiler used by MFEM. Get the compiler and the options for compiling
 # and linking from MFEM's config.mk. (Skip this if the target does not require
 # building.)
@@ -73,15 +85,16 @@ ifeq (,$(filter help clean distclean style,$(MAKECMDGOALS)))
 endif
 
 CXX = $(MFEM_CXX)
-CPPFLAGS = $(MFEM_CPPFLAGS)
+CPPFLAGS = $(MFEM_CPPFLAGS) $(CALIPER_FLAGS) $(ADIAK_FLAGS)
 CXXFLAGS = $(MFEM_CXXFLAGS)
 
 # MFEM config does not define C compiler
 CC     = gcc
 CFLAGS = -O3
 
 # Optional link flags
-LDFLAGS =
+LDFLAGS = $(CALIPER_LDFLAGS) $(ADIAK_LDFLAGS)
+
 
 OPTIM_OPTS = -O3
 DEBUG_OPTS = -g -Wall -std=c++11
@@ -102,7 +115,8 @@ ifeq ($(REMHOS_DEBUG),YES)
 endif
 
 LIBS = $(strip $(REMHOS_LIBS) $(LDFLAGS))
-CCC  = $(strip $(CXX) $(REMHOS_FLAGS))
+EXTRA_INC_DIR = $(or $(wildcard $(MFEM_DIR)/include/mfem),$(MFEM_DIR))
+CCC  = $(strip $(CXX) $(REMHOS_FLAGS) $(if $(EXTRA_INC_DIR),-I$(EXTRA_INC_DIR)))
 Ccc  = $(strip $(CC) $(CFLAGS) $(GL_OPTS))
 
 SOURCE_FILES = remhos.cpp remhos_tools.cpp remhos_lo.cpp remhos_ho.cpp \