From a2f76a0f51275df5773327e168bd90b61ce5244d Mon Sep 17 00:00:00 2001
From: bjpalmer <bruce.palmer@pnnl.gov>
Date: Fri, 13 Dec 2024 09:50:57 -0800
Subject: [PATCH] Capturing some debugging code.

---
 comex/src-mpi-pr/comex.c |  39 +++++-----
 global/testing/c_test.c  | 151 +++++++++++++++++++++++++++++++++++----
 2 files changed, 158 insertions(+), 32 deletions(-)

diff --git a/comex/src-mpi-pr/comex.c b/comex/src-mpi-pr/comex.c
index 36cc5326..125a2ded 100644
--- a/comex/src-mpi-pr/comex.c
+++ b/comex/src-mpi-pr/comex.c
@@ -2804,13 +2804,13 @@ int comex_malloc(void *ptrs[], size_t size, comex_group_t group)
                 reg_entries_local[reg_entries_local_count++] = reg_entries[i];
             }
         }
-        // else if (g_state.hostid[reg_entries[i].rank]
-        //         == g_state.hostid[my_world_rank]) 
+        else if (g_state.hostid[reg_entries[i].rank]
+                 == g_state.hostid[my_world_rank]) 
 
-        else if (g_state.master[reg_entries[i].rank] == 
-           g_state.master[get_my_master_rank_with_same_hostid(g_state.rank,
-           g_state.node_size, smallest_rank_with_same_hostid, largest_rank_with_same_hostid,
-           num_progress_ranks_per_node, is_node_ranks_packed)] )
+//        else if (g_state.master[reg_entries[i].rank] == 
+//           g_state.master[get_my_master_rank_with_same_hostid(g_state.rank,
+//           g_state.node_size, smallest_rank_with_same_hostid, largest_rank_with_same_hostid,
+//           num_progress_ranks_per_node, is_node_ranks_packed)] )
 #if 0
 #if MASTER_IS_SMALLEST_SMP_RANK
         else if (g_state.master[reg_entries[i].rank] ==
@@ -3215,6 +3215,9 @@ int comex_malloc_dev(void *ptrs[], size_t size, comex_group_t group)
                 reg_entries_local[reg_entries_local_count++] = reg_entries[i];
             }
         }
+//        else if (g_state.hostid[reg_entries[i].rank]
+//                 == g_state.hostid[my_world_rank]) 
+
         /* my master is the same as the master of the notifier */
         else if (g_state.master[reg_entries[i].rank] == 
            g_state.master[get_my_master_rank_with_same_hostid(g_state.rank,
@@ -7501,8 +7504,8 @@ STATIC void nb_put(void *src, void *dst, int bytes, int proc, nb_t *nb)
 
     if (COMEX_ENABLE_PUT_SMP) {
         /* put to SMP node */
-        // if (g_state.hostid[proc] == g_state.hostid[g_state.rank]) 
-        if (g_state.master[proc] == g_state.master[g_state.rank]) 
+        if (g_state.hostid[proc] == g_state.hostid[g_state.rank]) 
+//        if (g_state.master[proc] == g_state.master[g_state.rank]) 
         {
             reg_entry_t *reg_entry = NULL;
             void *mapped_offset = NULL;
@@ -7714,8 +7717,8 @@ STATIC void nb_get(void *src, void *dst, int bytes, int proc, nb_t *nb)
 
     if (COMEX_ENABLE_GET_SMP) {
         /* get from SMP node */
-        // if (g_state.hostid[proc] == g_state.hostid[g_state.rank]) 
-        if (g_state.master[proc] == g_state.master[g_state.rank]) 
+        if (g_state.hostid[proc] == g_state.hostid[g_state.rank]) 
+//        if (g_state.master[proc] == g_state.master[g_state.rank]) 
         {
             reg_entry_t *reg_entry = NULL;
             void *mapped_offset = NULL;
@@ -7905,8 +7908,8 @@ STATIC void nb_acc(int datatype, void *scale,
 
     if (COMEX_ENABLE_ACC_SMP) {
         /* acc to same SMP node */
-        // if (g_state.hostid[proc] == g_state.hostid[g_state.rank]) 
-        if (g_state.master[proc] == g_state.master[g_state.rank]) 
+        if (g_state.hostid[proc] == g_state.hostid[g_state.rank]) 
+//        if (g_state.master[proc] == g_state.master[g_state.rank]) 
         {
             reg_entry_t *reg_entry = NULL;
             void *mapped_offset = NULL;
@@ -9585,8 +9588,8 @@ STATIC void nb_putv(
               }
 #endif
         } else if (COMEX_ENABLE_PUT_SMP &&
-//          g_state.hostid[proc] == g_state.hostid[g_state.rank]) {
-          g_state.master[proc] == g_state.master[g_state.rank]) {
+          g_state.hostid[proc] == g_state.hostid[g_state.rank]) {
+//          g_state.master[proc] == g_state.master[g_state.rank]) {
           /* put to process on same SMP node */
           int j;
           void **src;
@@ -9918,8 +9921,8 @@ STATIC void nb_getv(
 #endif
         }
         else if (COMEX_ENABLE_GET_SMP && 
-//                   g_state.hostid[proc] == g_state.hostid[g_state.rank]) {
-                   g_state.master[proc] == g_state.master[g_state.rank]) {
+                   g_state.hostid[proc] == g_state.hostid[g_state.rank]) {
+//                   g_state.master[proc] == g_state.master[g_state.rank]) {
             int j;
             void **src;
             void **dst;
@@ -10237,8 +10240,8 @@ STATIC void nb_accv(
 #endif
         }
         else if (COMEX_ENABLE_PUT_SMP &&
-//                   g_state.hostid[proc] == g_state.hostid[g_state.rank]) {
-                   g_state.master[proc] == g_state.master[g_state.rank]) {
+                   g_state.hostid[proc] == g_state.hostid[g_state.rank]) {
+//                   g_state.master[proc] == g_state.master[g_state.rank]) {
             if (fence_array[g_state.master[proc]]) {
                 _fence_master(g_state.master[proc]);
             }
diff --git a/global/testing/c_test.c b/global/testing/c_test.c
index 1f184c53..5d99b495 100644
--- a/global/testing/c_test.c
+++ b/global/testing/c_test.c
@@ -166,7 +166,13 @@ void test_int_array(int on_device, int local_buf_on_device)
   double tbeg;
   double zero = 0.0;
   int ok;
-
+  int me, lprocs, lpdx, lpdy;
+ 
+  /* get local rank and default group size */
+  me = GA_Nodeid();
+  lprocs = GA_Nnodes();
+  factor(lprocs,&lpdx,&lpdy);
+  printf("p[%d] lprocs: %d lpdx: %d lpdy: %d\n",wrank,lprocs,lpdx,lpdy);
   tput = 0.0;
   tget = 0.0;
   tacc = 0.0;
@@ -182,29 +188,32 @@ void test_int_array(int on_device, int local_buf_on_device)
   ndim = 2;
   dims[0] = DIMSIZE;
   dims[1] = DIMSIZE;
-  xinc = DIMSIZE/pdx;
-  yinc = DIMSIZE/pdy;
-  ipx = rank%pdx;
-  ipy = (rank-ipx)/pdx;
-  isx = (ipx+1)%pdx;
-  isy = (ipy+1)%pdy;
+  xinc = DIMSIZE/lpdx;
+  yinc = DIMSIZE/lpdy;
+  ipx = me%lpdx;
+  ipy = (me-ipx)/lpdx;
+  isx = (ipx+1)%lpdx;
+  isy = (ipy+1)%lpdy;
   /* Guarantee some data exchange between nodes */
   lo[0] = isx*xinc;
   lo[1] = isy*yinc;
-  if (isx<pdx-1) {
+  if (isx<lpdx-1) {
     hi[0] = (isx+1)*xinc-1;
   } else {
     hi[0] = DIMSIZE-1;
   }
-  if (isy<pdy-1) {
+  if (isy<lpdy-1) {
     hi[1] = (isy+1)*yinc-1;
   } else {
     hi[1] = DIMSIZE-1;
   }
+  printf("p[%d] lo[0]: %d hi[0]: %d lo[1]: %d hi[i]: %d\n",
+      wrank,lo[0],hi[0],lo[1],hi[1]);
   nelem = (hi[0]-lo[0]+1)*(hi[1]-lo[1]+1);
 
   /* create a global array and initialize it to zero */
   tbeg = GA_Wtime();
+  printf("p[%d] Got to 1\n",wrank);
   g_a = NGA_Create_handle();
   NGA_Set_data(g_a, ndim, dims, C_INT);
   if (!on_device) {
@@ -214,6 +223,7 @@ void test_int_array(int on_device, int local_buf_on_device)
   NGA_Allocate(g_a);
   t_create += (GA_Wtime()-tbeg);
 
+  printf("p[%d] Got to 2\n",wrank);
   /* allocate a local buffer and initialize it with values*/
   nsize = (hi[0]-lo[0]+1)*(hi[1]-lo[1]+1);
   if (local_buf_on_device) {
@@ -227,8 +237,10 @@ void test_int_array(int on_device, int local_buf_on_device)
 
   for (n=0; n<NLOOP; n++) {
     tbeg = GA_Wtime();
+  printf("p[%d] Got to 3\n",wrank);
     GA_Zero(g_a);
     GA_Fill(g_a,&zero);
+  printf("p[%d] Got to 4\n",wrank);
     ld = (hi[1]-lo[1]+1);
     if (local_buf_on_device) {
       if (lo[0]<=hi[0] && lo[1]<=hi[1]) {
@@ -258,7 +270,9 @@ void test_int_array(int on_device, int local_buf_on_device)
     t_chk += (GA_Wtime()-tbeg);
     /* copy data to global array */
     tbeg = GA_Wtime();
+  printf("p[%d] Got to 5\n",wrank);
     NGA_Put(g_a, lo, hi, buf, &ld);
+  printf("p[%d] Got to 6\n",wrank);
     tput += (GA_Wtime()-tbeg);
     t_put += (GA_Wtime()-tbeg);
     put_cnt += nsize;
@@ -266,9 +280,10 @@ void test_int_array(int on_device, int local_buf_on_device)
     GA_Sync();
     t_sync += (GA_Wtime()-tbeg);
     tbeg = GA_Wtime();
-    NGA_Distribution(g_a,rank,tlo,thi);
+    NGA_Distribution(g_a,me,tlo,thi);
+  printf("p[%d] Got to 7\n",wrank);
 #if 1
-    if (rank == 0 && n == 0) printf("Completed NGA_Distribution\n");
+    if (me == 0 && n == 0) printf("Completed NGA_Distribution\n");
     if (tlo[0]<=thi[0] && tlo[1]<=thi[1]) {
       int tnelem = (thi[0]-tlo[0]+1)*(thi[1]-tlo[1]+1);
       if (on_device) {
@@ -306,6 +321,7 @@ void test_int_array(int on_device, int local_buf_on_device)
     GA_Sync();
     t_sync += (GA_Wtime()-tbeg);
 #endif
+  printf("p[%d] Got to 8\n",wrank);
 
     /* zero out local buffer */
     if (local_buf_on_device) {
@@ -509,8 +525,9 @@ void print_bw()
 {
   int iB = 1<<20;
   double rB = 1.0/((double)iB);
+  int me = GA_Nodeid();
   /*      12345678901234567890123456789012345678901234567890123456789012345 */
-  if (rank != 0) return;
+  if (me != 0) return;
 
   printf("///////////////////////////////////////////////////////////////\n");
   printf("//      Put (MB/sec) //     Get (MB/sec) //     Acc (MB/sec) //\n");
@@ -1825,7 +1842,7 @@ int main(int argc, char **argv) {
     printf("\n  1D arrays are of size %d\n",BLOCK1*nprocs);
     printf("\n  Number of loops in each test %d\n\n",NLOOP);
   }
-#if 1
+#if 0
   if (rank == 0) printf("  Testing integer array on device, local buffer on host\n");
   test_int_array(1,0);
   print_bw();
@@ -1908,7 +1925,6 @@ int main(int argc, char **argv) {
   test_dbl_1d_array(0,local_buf_on_device);
   print_bw();
 
-#endif
   if (rank == 0) printf("  Testing scatter/gather operations for double"
       " array\n  on device, local buffer on host\n");
   test_dbl_scatter(1,0);
@@ -1929,6 +1945,113 @@ int main(int argc, char **argv) {
   test_dbl_scatter(0,local_buf_on_device);
   print_bw();
 
+#endif
+#if 1
+  /* Check to see if code runs on subgroups */
+  /* Don't run this test on an odd number of processors */
+  if (nprocs%2 == 0) {
+    int grp_size = nprocs/2;
+    int grp, world;
+    int *llist, *devIDs;
+    int ndev, icnt;
+    int me;
+    if (rank == 0) printf("  Test  operations on subgroup\n");
+    free(list);
+    list = (int*)malloc(grp_size*sizeof(int));
+    for (i=0; i<grp_size; i++) {
+      if (rank < grp_size) {
+        list[i] = i;
+      } else {
+        list[i] = i + grp_size;
+      }
+    }
+    grp = GA_Pgroup_create(list, grp_size);
+    world = GA_Pgroup_get_default();
+    GA_Pgroup_set_default(grp);
+    me = GA_Nodeid();
+    printf("p[%d] rank: %d me: %d\n",wrank,rank,me);
+    printf("p[%d] world: %d grp_size: %d\n",wrank,world,grp_size);
+    llist = (int*)malloc(grp_size*sizeof(int));
+    devIDs = (int*)malloc(grp_size*sizeof(int));
+    GA_Device_host_list(llist, devIDs, &ndev, grp);
+    printf("p[%d] ndev: %d\n",wrank,ndev);
+    for (i=0; i<ndev; i++) {
+      printf("p[%d] list[%d]: %d devIDs[%d]: %d\n",wrank,i,llist[i],i,devIDs[i]);
+    }
+    free(llist);
+    free(devIDs);
+    if (ndev == grp_size) {
+      for (i=0; i<grp_size; i++) {
+        list[i] = i;
+      }
+#if 1
+      if (rank < grp_size) {
+        if (me == 0) printf("  Testing operations on group 1\n");
+#if 0
+        if (me == 0) printf("  Group 1: Testing integer array on"
+            " device, local buffer on host\n");
+        test_int_array(1,0);
+        print_bw();
+
+        if (me == 0) printf("  Group 1: Testing integer array on"
+            " device, local buffer on device\n");
+        test_int_array(1,local_buf_on_device);
+        print_bw();
+
+#endif
+        if (me == 0) printf("  Group 1: Testing integer array on"
+            " host, local buffer on host\n");
+        test_int_array(0,0);
+        print_bw();
+#if 0
+
+        if (me == 0) printf("  Group 1: Testing integer array on"
+            " host, local buffer on device\n");
+        test_int_array(0,local_buf_on_device);
+        print_bw();
+#endif
+      }
+#endif
+      fflush(stdout);
+      GA_Pgroup_sync(world);
+#if 1
+      if (rank >= grp_size) {
+        if (me == 0) printf("  Testing operations on group 2\n");
+#if 0
+        if (me == 0) printf("  Group 2: Testing integer array on"
+            " device, local buffer on host\n");
+        test_int_array(1,0);
+        print_bw();
+
+        if (me == 0) printf("  Group 2: Testing integer array on"
+            " device, local buffer on device\n");
+        test_int_array(1,local_buf_on_device);
+        print_bw();
+
+#endif
+        if (me == 0) printf("  Group 2: Testing integer array on"
+            " host, local buffer on host\n");
+        test_int_array(0,0);
+        print_bw();
+
+#if 0
+        if (me == 0) printf("  Group 2: Testing integer array on"
+            " host, local buffer on device\n");
+        test_int_array(0,local_buf_on_device);
+        print_bw();
+#endif
+      }
+#endif
+    } else {
+      printf("p[%d] Number of processors in group does not match"
+          " number of devices. Ndev: %d Grp_size: %d\n",rank,ndev,grp_size);
+    }
+    GA_Pgroup_set_default(world);
+    free(list);
+  }
+#endif
+
+
   t_tot = GA_Wtime()-tbeg;
   /* Print out timing stats */
   GA_Dgop(&t_put,1,"+");