From 804fb46f3c0cad06225768e765b972f6e196686a Mon Sep 17 00:00:00 2001 From: Bert Wesarg Date: Fri, 22 Jul 2022 08:37:47 +0200 Subject: [PATCH 1/5] Fix readme GitHub's Markdown render tries to find LaTeX `$`-math envs. Avoid this by using a verbatime block. --- README.md | 29 ++++++++++++++++------------- 1 file changed, 16 insertions(+), 13 deletions(-) diff --git a/README.md b/README.md index 57e8cebc..4f06b684 100644 --- a/README.md +++ b/README.md @@ -10,9 +10,8 @@ Redistribution and use in source and binary forms, with or without modification, THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS AND CONTRIBUTORS "AS IS" AND ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR PURPOSE ARE DISCLAIMED. IN NO EVENT SHALL THE COPYRIGHT HOLDER OR CONTRIBUTORS BE LIABLE FOR ANY DIRECT, INDIRECT, INCIDENTAL, SPECIAL, EXEMPLARY, OR CONSEQUENTIAL DAMAGES (INCLUDING, BUT NOT LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS OR SERVICES; LOSS OF USE, DATA, OR PROFITS; OR BUSINESS INTERRUPTION) HOWEVER CAUSED AND ON ANY THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT LIABILITY, OR TORT (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT OF THE USE OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE. -================================= -A hipified version of Quicksilver -================================= +# A hipified version of Quicksilver + NOTE: ===== This will only work with ROCM 1.7 and later. @@ -42,23 +41,27 @@ I have made several changes to get this to work with hip: Compiling and running: ====================== -To compile you must set several environment variables in the Makefile in the src/ directory. For example, in order to compile using HIP without MPI, and up to 15 sub-iterations for a particle per kernel launch, you could set the environment variables as follows (this is how they are set by default, you just need to point to where HIP is installed): +To compile you must set several environment variables in the Makefile in the `src/` directory. For example, in order to compile using HIP without MPI, and up to 15 sub-iterations for a particle per kernel launch, you could set the environment variables as follows (this is how they are set by default, you just need to point to where HIP is installed): -CXX = $(HIP)/bin/hipcc
-CXXFLAGS = -I$(HIP)/include/
-CPPFLAGS = -DHAVE_HIP=1 -DMaxIt=15
+``` +CXX = $(HIP)/bin/hipcc +CXXFLAGS = -I$(HIP)/include/ +CPPFLAGS = -DHAVE_HIP=1 -DMaxIt=15 LDFLAGS = -L$(HIP)/lib -L$(HIP)/lib +``` To compile using HIP and MPI and up to 15 sub-iterations for a particle per kernel launch, you could set the following environment variables: -CXX = $(HIP)/bin/hipcc
-CXXFLAGS1 = -I$(HIP)/include/
-CXXFLAGS2 = $(CXXFLAGS1) -I$(MPIPATH)/include
-CXXFLAGS = $(CXXFLAGS2) -pthread
-CPPFLAGS = -DHAVE_HIP=1 -DHAVE_MPI -DMaxIt=15
+``` +CXX = $(HIP)/bin/hipcc +CXXFLAGS1 = -I$(HIP)/include/ +CXXFLAGS2 = $(CXXFLAGS1) -I$(MPIPATH)/include +CXXFLAGS = $(CXXFLAGS2) -pthread +CPPFLAGS = -DHAVE_HIP=1 -DHAVE_MPI -DMaxIt=15 LDFLAGS = -L$(HIP)/lib -L$(MPIPATH)/lib -lmpicxx -lmpi +``` -To run look in src/READ.ME.HOW.TO.RUN. You can also look at the examples in the .sh files found in the directories 'Examples/CORAL2_Benchmark/Problem1' or 'Examples/CORAL2_Benchmark/Problem2'. +To run look in `src/READ.ME.HOW.TO.RUN`. You can also look at the examples in the `.sh` files found in the directories `Examples/CORAL2_Benchmark/Problem1` or `Examples/CORAL2_Benchmark/Problem2`. A Note on ROCm Versions: ======================== From 9a0921a67918c5175f316073e7a116ad24365cbd Mon Sep 17 00:00:00 2001 From: Bert Wesarg Date: Fri, 22 Jul 2022 08:22:55 +0200 Subject: [PATCH 2/5] NVTX: Fix memory leak --- src/NVTX_Range.hh | 1 + 1 file changed, 1 insertion(+) diff --git a/src/NVTX_Range.hh b/src/NVTX_Range.hh index 4eb4110d..cab8822f 100644 --- a/src/NVTX_Range.hh +++ b/src/NVTX_Range.hh @@ -30,6 +30,7 @@ class NVTX_Range char *result = strdup(rangeName.c_str()); _rangeId = nvtxRangeStartA(result); _isOpen = true; + free(result); #endif } From d25def3648cfa07175dec9c78ea4b7999915a169 Mon Sep 17 00:00:00 2001 From: Bert Wesarg Date: Fri, 22 Jul 2022 08:19:52 +0200 Subject: [PATCH 3/5] NVTX: Use scopes to avoid `endRange` --- src/NVTX_Range.hh | 2 +- src/main.cc | 16 +++++++++------- 2 files changed, 10 insertions(+), 8 deletions(-) diff --git a/src/NVTX_Range.hh b/src/NVTX_Range.hh index cab8822f..6f3f447b 100644 --- a/src/NVTX_Range.hh +++ b/src/NVTX_Range.hh @@ -42,6 +42,7 @@ class NVTX_Range #endif } + private: void endRange() { #ifdef USE_NVTX @@ -50,7 +51,6 @@ class NVTX_Range #endif } - private: #ifdef USE_NVTX nvtxRangeId_t _rangeId; bool _isOpen; diff --git a/src/main.cc b/src/main.cc index 903357c4..ab9784a9 100644 --- a/src/main.cc +++ b/src/main.cc @@ -392,16 +392,18 @@ void cycleTracking(MonteCarlo *monteCarlo, uint64_cu* tallies, uint64_cu * talli MC_FASTTIMER_START(MC_Fast_Timer::cycleTracking_MPI); - NVTX_Range collapseRange("cycleTracking_Collapse_ProcessingandProcessed"); - my_particle_vault.collapseProcessing(); - my_particle_vault.collapseProcessed(); - collapseRange.endRange(); + { + NVTX_Range collapseRange("cycleTracking_Collapse_ProcessingandProcessed"); + my_particle_vault.collapseProcessing(); + my_particle_vault.collapseProcessed(); + } //Test for done - blocking on all MPI ranks - NVTX_Range doneRange("cycleTracking_Test_Done_New"); - done = monteCarlo->particle_buffer->Test_Done_New( new_test_done_method ); - doneRange.endRange(); + { + NVTX_Range doneRange("cycleTracking_Test_Done_New"); + done = monteCarlo->particle_buffer->Test_Done_New( new_test_done_method ); + } MC_FASTTIMER_STOP(MC_Fast_Timer::cycleTracking_MPI); From 036044737213604197fe142691857369e072e0ca Mon Sep 17 00:00:00 2001 From: Bert Wesarg Date: Fri, 22 Jul 2022 08:30:36 +0200 Subject: [PATCH 4/5] NVTX: Switch to push/pop No need to use the more general start/end which can overlap and span threads. --- src/NVTX_Range.hh | 20 ++------------------ 1 file changed, 2 insertions(+), 18 deletions(-) diff --git a/src/NVTX_Range.hh b/src/NVTX_Range.hh index 6f3f447b..1f1f9783 100644 --- a/src/NVTX_Range.hh +++ b/src/NVTX_Range.hh @@ -28,8 +28,7 @@ class NVTX_Range { #ifdef USE_NVTX char *result = strdup(rangeName.c_str()); - _rangeId = nvtxRangeStartA(result); - _isOpen = true; + nvtxRangePushA(result); free(result); #endif } @@ -37,24 +36,9 @@ class NVTX_Range ~NVTX_Range() { #ifdef USE_NVTX - if (_isOpen) - nvtxRangeEnd(_rangeId); + nvtxRangePop(); #endif } - - private: - void endRange() - { - #ifdef USE_NVTX - nvtxRangeEnd(_rangeId); - _isOpen = false; - #endif - } - - #ifdef USE_NVTX - nvtxRangeId_t _rangeId; - bool _isOpen; - #endif }; #endif From 118b0d5f248c603d3e6c2c71e3ff1f21350bd47e Mon Sep 17 00:00:00 2001 From: Bert Wesarg Date: Fri, 22 Jul 2022 08:31:04 +0200 Subject: [PATCH 5/5] Add rocTx instrumentation --- src/Makefile | 3 +++ src/NVTX_Range.hh | 6 ++++++ 2 files changed, 9 insertions(+) diff --git a/src/Makefile b/src/Makefile index d28de381..72e2ed47 100644 --- a/src/Makefile +++ b/src/Makefile @@ -78,6 +78,9 @@ # -DUSE_NVTX Define this for some extra NVProf profiling information. # It will create regions that can be visualized in NVVP. # +# -DUSE_ROCTX Define this for some extra ROCtx profiling information. +# Needs -lroctx64 in LDFLAGS. +# # -DUSE_OPENMP_NO_GPU # Define this for runs with OPENMP_TARGET but when you do not # want to target GPU devices only the CPU diff --git a/src/NVTX_Range.hh b/src/NVTX_Range.hh index 1f1f9783..c96ae417 100644 --- a/src/NVTX_Range.hh +++ b/src/NVTX_Range.hh @@ -17,6 +17,8 @@ #ifdef USE_NVTX #include "nvToolsExt.h" +#elif defined(USE_ROCTX) +#include #endif @@ -30,6 +32,8 @@ class NVTX_Range char *result = strdup(rangeName.c_str()); nvtxRangePushA(result); free(result); + #elif defined(USE_ROCTX) + roctxRangePushA(rangeName.c_str()); #endif } @@ -37,6 +41,8 @@ class NVTX_Range { #ifdef USE_NVTX nvtxRangePop(); + #elif defined(USE_ROCTX) + roctxRangePop(); #endif } };