Merge pull request #433 from jacobmerson/pcu-object-breaking

Breaking changes with PCU.

- Ported PCU interface to not use global state
  - i.e., mesh creation requires passing in a PCU instance
- The new interface uses C++
- A C interface for PCU still exists in pcu_c.cc
  - Create an instance of the C handle struct "PCU_t"
  - The original pcu.c functions can be used by passing this struct
- Renamed source and headers files: 
  - PCU.h became PCU_C.h: PCU.h was repurposed as the C++ header
  - pcu.c became pcu_c.cc: pcu.c was deleted and pcu_c.cc created without history
  - more details on the history is here: #433 (comment)
- Build documentation (`make doc`) to generate doxygen based docs or see https://www.scorec.rpi.edu/pumi/doxygen/pcu.html for basic PCU.cc usage

CMakeLists.txt

@@ -6,7 +6,7 @@ endif()
 # This is the top level CMake file for the SCOREC build
 cmake_minimum_required(VERSION 3.8)
 
-project(SCOREC VERSION 2.2.10 LANGUAGES CXX C)
+project(SCOREC VERSION 2.3.0 LANGUAGES CXX C)
 
 include(cmake/bob.cmake)
 include(cmake/xsdk.cmake)

Doxyfile.in

@@ -793,7 +793,7 @@ INPUT                  = \
 @CMAKE_CURRENT_SOURCE_DIR@/gmi/gmi_mesh.h \
 @CMAKE_CURRENT_SOURCE_DIR@/gmi/gmi_analytic.h \
 @CMAKE_CURRENT_SOURCE_DIR@/gmi/gmi_null.h \
-@CMAKE_CURRENT_SOURCE_DIR@/pcu/pcu.c \
+@CMAKE_CURRENT_SOURCE_DIR@/pcu/PCU.cc \
 @CMAKE_CURRENT_SOURCE_DIR@/mth/mth.dox \
 @CMAKE_CURRENT_SOURCE_DIR@/mth/mth.h \
 @CMAKE_CURRENT_SOURCE_DIR@/mth/mthVector.h \

Doxyfile_internal.in

@@ -826,7 +826,7 @@ INPUT                  = \
 @CMAKE_CURRENT_SOURCE_DIR@/gmi/gmi_mesh.h \
 @CMAKE_CURRENT_SOURCE_DIR@/gmi/gmi_analytic.h \
 @CMAKE_CURRENT_SOURCE_DIR@/gmi/gmi_null.h \
-@CMAKE_CURRENT_SOURCE_DIR@/pcu/pcu.c \
+@CMAKE_CURRENT_SOURCE_DIR@/pcu/PCU.cc \
 @CMAKE_CURRENT_SOURCE_DIR@/mth/mth.dox \
 @CMAKE_CURRENT_SOURCE_DIR@/mth/mth.h \
 @CMAKE_CURRENT_SOURCE_DIR@/mth/mthVector.h \

apf/apf.h

@@ -12,6 +12,10 @@
 #include "apfNew.h"
 #include "apfDynamicArray.h"
 
+namespace pcu{
+  class PCU;
+}
+
 #include <vector>
 #include <map>
 #include <limits>
@@ -460,7 +464,7 @@ class Integrator
       * process-local integrations into a global mesh integration,
       * if that is the user's goal.
       */
-    virtual void parallelReduce();
+    virtual void parallelReduce(pcu::PCU*);
   protected:
     int order;
     int ipnode;

apf/apfCGNS.cc

@@ -5,15 +5,13 @@
  * BSD license as described in the LICENSE file in the top-level directory.
  */
 
-#include <PCU.h>
 #include "apfMesh.h"
 #include "apfNumbering.h"
 #include "apfNumberingClass.h"
 #include "apfShape.h"
 #include "apfFieldData.h"
 #include <pcu_util.h>
 #include <lionPrint.h>
-//
 #include <sstream>
 #include <fstream>
 #include <iostream>
@@ -47,7 +45,7 @@ static Count count(apf::Mesh *m, int dim)
 {
   const int local = apf::countOwned(m, dim);
   int total = local;
-  PCU_Add_Ints(&total, 1); // size of total array
+  m->getPCU()->Add<int>(&total, 1); // size of total array
   return std::make_pair(total, local);
 }
 
@@ -290,7 +288,7 @@ void WriteFields(const CGNS &cgns, const std::vector<std::vector<apf::MeshEntity
     }
 
     int size = data.size();
-    PCU_Add_Ints(&size, 1); // size of total array
+    m->getPCU()->Add<int>(&size, 1); // size of total array
 
     // oddness of the api
     rmin[1] = rmin[0];
@@ -505,7 +503,7 @@ CellElementReturn WriteElements(const CGNS &cgns, apf::Mesh *m, apf::GlobalNumbe
     m->end(cellIter);
     numbersByElementType[o] = counter;
     int total = counter;
-    PCU_Add_Ints(&total, 1); // size of total array
+    m->getPCU()->Add<int>(&total, 1); // size of total array
     globalNumbersByElementType[o] = total;
   }
   cgsize_t allTotal = std::accumulate(globalNumbersByElementType.begin(), globalNumbersByElementType.end(), 0);
@@ -546,12 +544,12 @@ CellElementReturn WriteElements(const CGNS &cgns, apf::Mesh *m, apf::GlobalNumbe
       if (cgp_section_write(cgns.index, cgns.base, cgns.zone, name.c_str(), cgnsElementOrder[o], globalStart, globalEnd, 0, &sectionNumber)) // global start, end within the file for that element type
         cgp_error_exit();
 
-      std::vector<int> allNumbersForThisType(PCU_Comm_Peers(), 0);
+      std::vector<int> allNumbersForThisType(m->getPCU()->Peers(), 0);
       MPI_Allgather(&numbersByElementType[o], 1, MPI_INT, allNumbersForThisType.data(), 1,
-                    MPI_INT, PCU_Get_Comm());
+                    MPI_INT, m->getPCU()->GetMPIComm());
 
       cgsize_t num = 0;
-      for (int i = 0; i < PCU_Comm_Self(); i++)
+      for (int i = 0; i < m->getPCU()->Self(); i++)
         num += allNumbersForThisType[i];
 
       cgsize_t elStart = globalStart + num;
@@ -635,7 +633,7 @@ void AddBocosToMainBase(const CGNS &cgns, const CellElementReturn &cellResults,
       int startOfBCBlock = startingLocation + 1;
       const int number = bc.second.size();
       int total = number;
-      PCU_Add_Ints(&total, 1); // size of total array
+      m->getPCU()->Add<int>(&total, 1); // size of total array
       if (total > 0)
       {
         const auto allEnd = startOfBCBlock + total - 1; //one-based
@@ -657,12 +655,12 @@ void AddBocosToMainBase(const CGNS &cgns, const CellElementReturn &cellResults,
           }
         }
 
-        std::vector<int> allNumbersForThisType(PCU_Comm_Peers(), 0);
+        std::vector<int> allNumbersForThisType(m->getPCU()->Peers(), 0);
         MPI_Allgather(&number, 1, MPI_INT, allNumbersForThisType.data(), 1,
-                      MPI_INT, PCU_Get_Comm());
+                      MPI_INT, m->getPCU()->GetMPIComm());
 
         cgsize_t num = 0;
-        for (int i = 0; i < PCU_Comm_Self(); i++)
+        for (int i = 0; i < m->getPCU()->Self(); i++)
           num += allNumbersForThisType[i];
 
         cgsize_t elStart = startOfBCBlock + num;
@@ -685,34 +683,34 @@ void AddBocosToMainBase(const CGNS &cgns, const CellElementReturn &cellResults,
         cacheEnd = allEnd;
       }
     }
-    std::vector<int> cacheStarts(PCU_Comm_Peers(), 0);
+    std::vector<int> cacheStarts(m->getPCU()->Peers(), 0);
     MPI_Allgather(&cacheStart, 1, MPI_INT, cacheStarts.data(), 1,
-                  MPI_INT, PCU_Get_Comm());
-    std::vector<int> cacheEnds(PCU_Comm_Peers(), 0);
+                  MPI_INT, m->getPCU()->GetMPIComm());
+    std::vector<int> cacheEnds(m->getPCU()->Peers(), 0);
     MPI_Allgather(&cacheEnd, 1, MPI_INT, cacheEnds.data(), 1,
-                  MPI_INT, PCU_Get_Comm());
+                  MPI_INT, m->getPCU()->GetMPIComm());
     return std::make_pair(cacheStarts, cacheEnds);
   };
 
-  const auto globalElementList = [](const std::vector<cgsize_t> &bcList, std::vector<cgsize_t> &allElements) {
-    std::vector<int> sizes(PCU_Comm_Peers(), 0); // important initialiser
+  const auto globalElementList = [&m](const std::vector<cgsize_t> &bcList, std::vector<cgsize_t> &allElements) {
+    std::vector<int> sizes(m->getPCU()->Peers(), 0); // important initialiser
     const int l = bcList.size();
     MPI_Allgather(&l, 1, MPI_INT, sizes.data(), 1,
-                  MPI_INT, PCU_Get_Comm());
+                  MPI_INT, m->getPCU()->GetMPIComm());
 
     int totalLength = 0;
     for (const auto &i : sizes)
       totalLength += i;
 
-    std::vector<int> displacement(PCU_Comm_Peers(), -1);
+    std::vector<int> displacement(m->getPCU()->Peers(), -1);
     displacement[0] = 0;
     for (std::size_t i = 1; i < displacement.size(); i++)
       displacement[i] = displacement[i - 1] + sizes[i - 1];
 
     allElements.resize(totalLength);
     MPI_Allgatherv(bcList.data(), bcList.size(), MPI_INT, allElements.data(),
                    sizes.data(), displacement.data(), MPI_INT,
-                   PCU_Get_Comm());
+                   m->getPCU()->GetMPIComm());
   };
 
   const auto doVertexBC = [&](const auto &iter) {
@@ -755,7 +753,7 @@ void AddBocosToMainBase(const CGNS &cgns, const CellElementReturn &cellResults,
     for (const auto &p : iter->second)
     {
       const auto se = BCEntityAdder(EdgeLoop, p, startingLocation);
-      for (int i = 0; i < PCU_Comm_Peers(); i++)
+      for (int i = 0; i < m->getPCU()->Peers(); i++)
       {
         PCU_ALWAYS_ASSERT_VERBOSE(se.first[i] == se.first[0], "Must all be the same ");
         PCU_ALWAYS_ASSERT_VERBOSE(se.second[i] == se.second[0], "Must all be the same ");
@@ -779,7 +777,7 @@ void AddBocosToMainBase(const CGNS &cgns, const CellElementReturn &cellResults,
     {
       const auto se = BCEntityAdder(FaceLoop, p, startingLocation);
 
-      for (int i = 0; i < PCU_Comm_Peers(); i++)
+      for (int i = 0; i < m->getPCU()->Peers(); i++)
       {
         PCU_ALWAYS_ASSERT_VERBOSE(se.first[i] == se.first[0], "Must all be the same ");
         PCU_ALWAYS_ASSERT_VERBOSE(se.second[i] == se.second[0], "Must all be the same ");
@@ -1011,7 +1009,7 @@ void WriteCGNS(const char *prefix, apf::Mesh *m, const apf::CGNSBCMap &cgnsBCMap
 {
   static_assert(std::is_same<cgsize_t, int>::value, "cgsize_t not compiled as int");
 
-  const auto myRank = PCU_Comm_Self();
+  const auto myRank = m->getPCU()->Self();
   const Count vertexCount = count(m, 0);
   const Count edgeCount = count(m, 1);
   const Count faceCount = count(m, 2);
@@ -1031,7 +1029,7 @@ void WriteCGNS(const char *prefix, apf::Mesh *m, const apf::CGNSBCMap &cgnsBCMap
   //   PCU_Barrier();
   // }
 
-  PCU_Barrier();
+  m->getPCU()->Barrier();
   if (myRank == 0)
   {
     std::cout << "*******Global Mesh Stats*****************\n";
@@ -1048,7 +1046,7 @@ void WriteCGNS(const char *prefix, apf::Mesh *m, const apf::CGNSBCMap &cgnsBCMap
   sizes[2] = 0;                 // nodes are unsorted, as defined by api
 
   // Copy communicator
-  auto communicator = PCU_Get_Comm();
+  auto communicator = m->getPCU()->GetMPIComm();
   cgp_mpi_comm(communicator);
   //
   cgp_pio_mode(CGP_INDEPENDENT);

apf/apfCavityOp.cc

@@ -5,7 +5,6 @@
  * BSD license as described in the LICENSE file in the top-level directory.
  */
 
-#include <PCU.h>
 #include "apfCavityOp.h"
 #include "apf.h"
 #include "apfMesh2.h"
@@ -134,10 +133,10 @@ bool CavityOp::requestLocality(MeshEntity** entities, int count)
 
 bool CavityOp::sendPullRequests(std::vector<PullRequest>& received)
 {
-  int done = PCU_Min_Int(requests.empty());
+  int done = mesh->getPCU()->Min<int>(requests.empty());
   if (done) return false;
   /* throw in the local pull requests */
-  int self = PCU_Comm_Self();
+  int self = mesh->getPCU()->Self();
   received.reserve(requests.size());
   APF_ITERATE(Requests,requests,it)
   {
@@ -147,7 +146,7 @@ bool CavityOp::sendPullRequests(std::vector<PullRequest>& received)
     received.push_back(request);
   }
   /* now communicate the rest */
-  PCU_Comm_Begin();
+  mesh->getPCU()->Begin();
   APF_ITERATE(Requests,requests,it)
   {
     CopyArray remotes;
@@ -156,18 +155,17 @@ bool CavityOp::sendPullRequests(std::vector<PullRequest>& received)
     {
       int remotePart = rit->peer;
       MeshEntity* remoteEntity = rit->entity;
-      PCU_COMM_PACK(remotePart,remoteEntity);
-    }
+      mesh->getPCU()->Pack(remotePart,remoteEntity);    }
   }
   requests.clear();
-  PCU_Comm_Send();
-  while (PCU_Comm_Listen())
+  mesh->getPCU()->Send();
+  while (mesh->getPCU()->Listen())
   {
     PullRequest request;
-    request.to = PCU_Comm_Sender();
-    while ( ! PCU_Comm_Unpacked())
+    request.to = mesh->getPCU()->Sender();
+    while ( ! mesh->getPCU()->Unpacked())
     {
-      PCU_COMM_UNPACK(request.e);
+      mesh->getPCU()->Unpack(request.e);
       received.push_back(request);
     }
   }