Add global comm

.github/workflows/clang-tidy.yml

@@ -165,7 +165,7 @@ jobs:
       with:
         repo-name: 'redev-openmpi'
         repo-path: 'SCOREC/redev'
-        repo-ref: 'ac09848a5f9b89493e8b679c9080b9efe5538376'
+        repo-ref: 'eb52569702864979f5b7d03d9c082c96f20b2bd5'
         cache: true
         options: '-DCMAKE_CXX_COMPILER=`which mpicxx`
                   -DMPIEXEC_EXECUTABLE=`which mpirun`

.github/workflows/cmake-test.yml

@@ -165,7 +165,7 @@ jobs:
       with:
         repo-name: 'redev'
         repo-path: 'SCOREC/redev'
-        repo-ref: 'ac09848a5f9b89493e8b679c9080b9efe5538376'
+        repo-ref: 'eb52569702864979f5b7d03d9c082c96f20b2bd5'
         cache: true
         cache-suffix: ${{ matrix.python_api == 'ON' && '-shared' || '' }}
         options: '-DCMAKE_CXX_COMPILER=`which mpicxx`

.github/workflows/self-hosted.yml

@@ -111,7 +111,7 @@ jobs:
           # redev
           git clone https://github.com/SCOREC/redev.git ${workDir}/redev
           cd ${workDir}/redev
-          git checkout ac09848a5f9b89493e8b679c9080b9efe5538376
+          git checkout eb52569702864979f5b7d03d9c082c96f20b2bd5
           cd $workDir
           rdbdir=${workDir}/build-redev
           cmake -S ${workDir}/redev -B $rdbdir \

src/pcms/coupler/CMakeLists.txt

@@ -7,6 +7,7 @@ set(PCMS_COUPLER_HEADERS
         field_exchange_planner.h
         partition.h
         overlap_mask.h
+        global_communicator.h
 )
 
 

src/pcms/coupler/coupler.hpp

@@ -11,10 +11,55 @@
 #include "pcms/utility/assert.h"
 #include "pcms/utility/common.h"
 #include "pcms/utility/profile.h"
+#include "pcms/coupler/global_communicator.h"
 #include <memory>
 
 namespace pcms
 {
+template <typename T>
+class GlobalDataInterface
+{
+public:
+  GlobalDataInterface(const std::string& name, MPI_Comm mpi_comm,
+                      redev::Channel& channel)
+    : mpi_comm_(mpi_comm), comm_(name, mpi_comm_, channel)
+  {
+    PCMS_FUNCTION_TIMER;
+  }
+  // Todo: make a constant view
+  void SendData(Rank1View<T, pcms::HostMemorySpace> msg,
+                std::string variable_name,
+                redev::Mode mode = redev::Mode::Synchronous)
+  {
+    PCMS_FUNCTION_TIMER;
+
+    comm_.Send(msg.data_handle(), std::move(variable_name),
+               static_cast<std::size_t>(msg.extent(0)), mode);
+  }
+
+  void ReceiveData(Rank1View<T, pcms::HostMemorySpace> msg,
+                   std::string variable_name,
+                   redev::Mode mode = redev::Mode::Synchronous)
+  {
+    PCMS_FUNCTION_TIMER;
+
+    auto received = comm_.Receive(
+      std::move(variable_name), static_cast<std::size_t>(msg.extent(0)), mode);
+
+    PCMS_ALWAYS_ASSERT(received.size() ==
+                       static_cast<std::size_t>(msg.extent(0)));
+
+    std::copy(received.begin(), received.end(), msg.data_handle());
+  }
+
+private:
+  MPI_Comm mpi_comm_;
+  GlobalCommunicator<T> comm_;
+};
+using GlobalDataVariant =
+  std::variant<GlobalDataInterface<int8_t>, GlobalDataInterface<int32_t>,
+               GlobalDataInterface<int64_t>, GlobalDataInterface<float>,
+               GlobalDataInterface<double>>;
 
 class Application;
 
@@ -31,6 +76,25 @@ class FieldHandle
   void Receive(redev::Mode mode = redev::Mode::Synchronous) const;
   [[nodiscard]] Field<T>& GetField() const;
 
+private:
+  Application* app_;
+  std::string name_;
+};
+template <typename T>
+class DataHandle
+{
+public:
+  DataHandle(Application* app, std::string name)
+    : app_(app), name_(std::move(name))
+  {
+  }
+  void Send(Rank1View<T, pcms::HostMemorySpace> msg, std::string variable_name,
+            redev::Mode mode = redev::Mode::Synchronous) const;
+
+  void Receive(Rank1View<T, pcms::HostMemorySpace> msg,
+               std::string variable_name,
+               redev::Mode mode = redev::Mode::Synchronous) const;
+
 private:
   Application* app_;
   std::string name_;
@@ -70,7 +134,11 @@ class Application
   FieldHandle<T> AddField(std::string name, Field<T>&& field,
                           std::unique_ptr<FieldSerializer<T>> serializer,
                           bool participates = true);
-
+  template <typename T>
+  DataHandle<T> AddData(std::string name, MPI_Comm mpi_comm);
+  template <typename T>
+  [[nodiscard]] GlobalDataInterface<T>& GetDataInterface(
+    const std::string& name);
   void SendField(const std::string& name,
                  redev::Mode mode = redev::Mode::Synchronous)
   {
@@ -91,6 +159,29 @@ class Application
       [mode](auto& field_communicator) { field_communicator->Receive(); },
       detail::find_or_error(name, field_communicators_));
   };
+  template <typename T>
+  void SendData(const std::string& name,
+                Rank1View<T, pcms::HostMemorySpace> msg,
+                std::string variable_name,
+                redev::Mode mode = redev::Mode::Synchronous)
+  {
+    PCMS_FUNCTION_TIMER;
+    PCMS_ALWAYS_ASSERT(InSendPhase());
+
+    GetDataInterface<T>(name).SendData(msg, std::move(variable_name), mode);
+  }
+
+  template <typename T>
+  void ReceiveData(const std::string& name,
+                   Rank1View<T, pcms::HostMemorySpace> msg,
+                   std::string variable_name,
+                   redev::Mode mode = redev::Mode::Synchronous)
+  {
+    PCMS_FUNCTION_TIMER;
+    PCMS_ALWAYS_ASSERT(InReceivePhase());
+
+    GetDataInterface<T>(name).ReceiveData(msg, std::move(variable_name), mode);
+  }
   [[nodiscard]] bool InSendPhase() const noexcept
   {
     PCMS_FUNCTION_TIMER;
@@ -160,8 +251,49 @@ class Application
   std::map<const FieldLayout*, std::unique_ptr<FieldLayoutCommunicator>>
     field_layout_communicators_;
   std::map<std::string, std::unique_ptr<OverlapMask>> layout_overlap_masks_;
+  std::map<std::string, GlobalDataVariant> global_data_interfaces_;
 };
 
+template <typename T>
+DataHandle<T> Application::AddData(std::string name, MPI_Comm mpi_comm)
+{
+  PCMS_FUNCTION_TIMER;
+  auto [it, inserted] = global_data_interfaces_.try_emplace(
+    name, std::in_place_type<GlobalDataInterface<T>>, name, mpi_comm, channel_);
+  if (!inserted) {
+    throw pcms_error("Global data interface with this name already exists");
+  }
+  return DataHandle<T>{this, std::move(name)};
+}
+template <typename T>
+GlobalDataInterface<T>& Application::GetDataInterface(const std::string& name)
+{
+  auto* data_interface = std::get_if<GlobalDataInterface<T>>(
+    &detail::find_or_error(name, global_data_interfaces_));
+  if (data_interface == nullptr) {
+    throw pcms_error(
+      "Global data interface stored with different type than requested");
+  }
+  return *data_interface;
+}
+
+template <typename T>
+void DataHandle<T>::Send(Rank1View<T, pcms::HostMemorySpace> msg,
+                         std::string variable_name, redev::Mode mode) const
+{
+  PCMS_ALWAYS_ASSERT(app_ != nullptr);
+
+  app_->SendData<T>(name_, msg, std::move(variable_name), mode);
+}
+template <typename T>
+void DataHandle<T>::Receive(Rank1View<T, pcms::HostMemorySpace> msg,
+                            std::string variable_name, redev::Mode mode) const
+{
+  PCMS_ALWAYS_ASSERT(app_ != nullptr);
+
+  app_->ReceiveData<T>(name_, msg, std::move(variable_name), mode);
+}
+
 class Coupler
 {
 private:

src/pcms/coupler/global_communicator.h

@@ -0,0 +1,51 @@
+#ifndef PCMS_GLOBAL_COMMUNICATOR_H
+#define PCMS_GLOBAL_COMMUNICATOR_H
+#include <redev.h>
+#include <pcms/utility/profile.h>
+namespace pcms
+{
+using redev::Mode;
+template <typename T>
+struct GlobalCommunicator
+{
+  using value_type = T;
+
+public:
+  GlobalCommunicator(std::string name, MPI_Comm mpi_comm,
+                     redev::Channel& channel)
+    : mpi_comm(mpi_comm), channel_(channel), name_(std::move(name))
+  {
+    PCMS_FUNCTION_TIMER;
+    comm_ = channel_.CreateComm<T>(name_, mpi_comm, redev::CommType::Global);
+  }
+  GlobalCommunicator(const GlobalCommunicator&) = delete;
+  GlobalCommunicator& operator=(const GlobalCommunicator&) = delete;
+  GlobalCommunicator(GlobalCommunicator&&) = default;
+  GlobalCommunicator& operator=(GlobalCommunicator&&) = default;
+
+  void Send(T* msg, std::string VarName, size_t msg_size,
+            Mode mode = Mode::Synchronous)
+  {
+    PCMS_FUNCTION_TIMER;
+    PCMS_ALWAYS_ASSERT(channel_.InSendCommunicationPhase());
+    comm_.SetCommParams(VarName, msg_size);
+    comm_.Send(msg, mode);
+  }
+  std::vector<T> Receive(std::string VarName, size_t msg_size,
+                         Mode mode = Mode::Synchronous)
+  {
+    PCMS_FUNCTION_TIMER;
+    PCMS_ALWAYS_ASSERT(channel_.InReceiveCommunicationPhase());
+    comm_.SetCommParams(VarName, msg_size);
+    auto data = comm_.Recv(mode);
+    return data;
+  }
+
+private:
+  MPI_Comm mpi_comm;
+  redev::Channel& channel_;
+  std::string name_;
+  redev::BidirectionalComm<T> comm_;
+};
+} // namespace pcms
+#endif // PCMS_GLOBAL_COMMUNICATOR_H

test/CMakeLists.txt

@@ -96,6 +96,37 @@ if(PCMS_ENABLE_OMEGA_H)
       ${d3d16p}
       ignored)
   endif()
+  add_exe(test_GDI)
+  tri_mpi_test(
+          TESTNAME
+          test_GDI
+          TIMEOUT
+          20
+          NAME1
+          app
+          EXE1
+          ./test_GDI
+          PROCS1
+          1
+          ARGS1
+          1
+          NAME2
+          rdv
+          EXE2
+          ./test_GDI
+          PROCS2
+          1
+          ARGS2
+          -1
+          NAME3
+          app
+          EXE3
+          ./test_GDI
+          PROCS3
+          1
+          ARGS3
+          0
+  )
 
   set(d3d8p ${PCMS_TEST_DATA_DIR}/d3d/d3d-full_9k_sfc_p8.osh/)
   add_exe(test_twoClientOverlap)
@@ -380,7 +411,6 @@ if(Catch2_FOUND)
       APPEND
       PCMS_UNIT_TEST_SOURCES
       test_error_handling.cpp
-      test_eqdsk.cpp
       test_uniform_grid.cpp
       test_field_evaluation.cpp
       test_field_interpolation.cpp
@@ -401,7 +431,6 @@ if(Catch2_FOUND)
       test_omega_h_lagrange_field.cpp
       test_point_evaluator.cpp)
   endif()
-
   if(PCMS_ENABLE_MESHFIELDS)
     list(APPEND PCMS_UNIT_TEST_SOURCES
       test_load_vector.cpp)
@@ -426,13 +455,6 @@ if(Catch2_FOUND)
     target_link_libraries(unit_tests PRIVATE PETSc::PETSc)
   endif()
 
-  target_link_libraries(unit_tests PUBLIC 
-    Catch2::Catch2 
-    pcms::core
-    pcms_transfer
-    pcms_transfer
-  )
-
  target_include_directories(unit_tests PUBLIC ${CMAKE_CURRENT_SOURCE_DIR})
 
   add_executable(test_interpolation_on_ltx_mesh test_interpolation_on_ltx_mesh.cpp)