From b3841fb21eb589426fc594b071ef20caaab92797 Mon Sep 17 00:00:00 2001
From: David Bold <dave@ipp.mpg.de>
Date: Fri, 31 Jan 2025 14:51:11 +0100
Subject: [PATCH] make fci_comm openmp thread safe

Using a local set for each thread ensures we do not need a mutex for
adding data, at the cost of having to merge the different sets later.
---
 src/mesh/parallel/fci_comm.hxx | 26 ++++++++++++++++++++++++--
 1 file changed, 24 insertions(+), 2 deletions(-)

diff --git a/src/mesh/parallel/fci_comm.hxx b/src/mesh/parallel/fci_comm.hxx
index df08a02dda..40c1fe9f72 100644
--- a/src/mesh/parallel/fci_comm.hxx
+++ b/src/mesh/parallel/fci_comm.hxx
@@ -118,11 +118,30 @@ public:
         g2ly(mesh->ystart, mesh->getNYPE(), mesh->LocalNy, true),
         g2lz(mesh->zstart, 1, mesh->LocalNz, true),
         xyzl(g2lx.localwith, g2ly.localwith, g2lz.localwith),
-        xyzg(g2lx.globalwith, g2ly.globalwith, g2lz.globalwith), comm(BoutComm::get()) {};
-  void get(IndG3D ind) { ids.emplace(ind.ind); }
+        xyzg(g2lx.globalwith, g2ly.globalwith, g2lz.globalwith), comm(BoutComm::get()) {
+#ifdef _OPENMP
+    o_ids.resize(omp_get_max_threads());
+#endif
+  };
+  void get(IndG3D ind) {
+    ASSERT2(is_setup == false);
+#ifdef _OPENMP
+    ASSERT2(o_ids.size() > static_cast<size_t>(omp_get_thread_num()));
+    o_ids[omp_get_thread_num()].emplace(ind.ind);
+#else
+    ids.emplace(ind.ind);
+#endif
+  }
+
   void operator[](IndG3D ind) { return get(ind); }
   void setup() {
     ASSERT2(is_setup == false);
+#ifdef _OPENMP
+    for (auto& o_id : o_ids) {
+      ids.merge(o_id);
+    }
+    o_ids.clear();
+#endif
     toGet.resize(g2lx.npe * g2ly.npe * g2lz.npe);
     for (const auto id : ids) {
       IndG3D gind{id, g2ly.globalwith, g2lz.globalwith};
@@ -226,6 +245,9 @@ private:
     }
   }
   Mesh* mesh;
+#ifdef _OPENMP
+  std::vector<std::set<int>> o_ids;
+#endif
   std::set<int> ids;
   std::map<int, int> mapping;
   bool is_setup{false};