In EXAMPLE/ global interface drivers, ask P0 generates random Xtrue a…

…nd RHS, then broadcast to every other processes
eromero-vlc · Nov 2, 2022 · 3859112 · 3859112
1 parent 324d65f
commit 3859112
Show file tree

Hide file tree

Showing 17 changed files with 239 additions and 77 deletions.
diff --git a/EXAMPLE/pddrive1_ABglobal.c b/EXAMPLE/pddrive1_ABglobal.c
@@ -159,8 +159,18 @@ int main(int argc, char *argv[])
     *trans = 'N';
     ldx = n;
     ldb = m;
-    dGenXtrue_dist(n, nrhs, xtrue, ldx);
-    dFillRHS_dist(trans, nrhs, xtrue, ldx, &A, b, ldb);
+
+    if ( iam==0 ) {
+        dGenXtrue_dist(n, nrhs, xtrue, ldx);
+        dFillRHS_dist(trans, nrhs, xtrue, ldx, &A, b, ldb);
+
+        MPI_Bcast( xtrue, n*nrhs, MPI_DOUBLE, 0, grid.comm );
+        MPI_Bcast( b, m*nrhs, MPI_DOUBLE, 0, grid.comm );
+    } else {
+        MPI_Bcast( xtrue, n*nrhs, MPI_DOUBLE, 0, grid.comm );
+        MPI_Bcast( b, m*nrhs, MPI_DOUBLE, 0, grid.comm );
+    }
+
     for (j = 0; j < nrhs; ++j)
 	for (i = 0; i < m; ++i) b1[i+j*ldb] = b[i+j*ldb];
 

diff --git a/EXAMPLE/pddrive2_ABglobal.c b/EXAMPLE/pddrive2_ABglobal.c
@@ -157,8 +157,17 @@ int main(int argc, char *argv[])
     *trans = 'N';
     ldx = n;
     ldb = m;
-    dGenXtrue_dist(n, nrhs, xtrue, ldx);
-    dFillRHS_dist(trans, nrhs, xtrue, ldx, &A, b, ldb);
+
+    if ( iam==0 ) {
+        dGenXtrue_dist(n, nrhs, xtrue, ldx);
+        dFillRHS_dist(trans, nrhs, xtrue, ldx, &A, b, ldb);
+
+        MPI_Bcast( xtrue, n*nrhs, MPI_DOUBLE, 0, grid.comm );
+        MPI_Bcast( b, m*nrhs, MPI_DOUBLE, 0, grid.comm );
+    } else {
+        MPI_Bcast( xtrue, n*nrhs, MPI_DOUBLE, 0, grid.comm );
+        MPI_Bcast( b, m*nrhs, MPI_DOUBLE, 0, grid.comm );
+    }
 
     /* Save a copy of the right-hand side. */  
     if ( !(b1 = doubleMalloc_dist(m * nrhs)) ) ABORT("Malloc fails for b1[]");

diff --git a/EXAMPLE/pddrive3_ABglobal.c b/EXAMPLE/pddrive3_ABglobal.c
@@ -163,8 +163,17 @@ int main(int argc, char *argv[])
     *trans = 'N';
     ldx = n;
     ldb = m;
-    dGenXtrue_dist(n, nrhs, xtrue, ldx);
-    dFillRHS_dist(trans, nrhs, xtrue, ldx, &A, b, ldb);
+
+    if ( iam==0 ) {
+        dGenXtrue_dist(n, nrhs, xtrue, ldx);
+        dFillRHS_dist(trans, nrhs, xtrue, ldx, &A, b, ldb);
+
+        MPI_Bcast( xtrue, n*nrhs, MPI_DOUBLE, 0, grid.comm );
+        MPI_Bcast( b, m*nrhs, MPI_DOUBLE, 0, grid.comm );
+    } else {
+        MPI_Bcast( xtrue, n*nrhs, MPI_DOUBLE, 0, grid.comm );
+        MPI_Bcast( b, m*nrhs, MPI_DOUBLE, 0, grid.comm );
+    }
 
     /* Save a copy of the right-hand side. */  
     if ( !(b1 = doubleMalloc_dist(m * nrhs)) ) ABORT("Malloc fails for b1[]");

diff --git a/EXAMPLE/pddrive4_ABglobal.c b/EXAMPLE/pddrive4_ABglobal.c
@@ -177,8 +177,17 @@ int main(int argc, char *argv[])
 	*trans = 'N';
 	ldx = n;
 	ldb = m;
-	dGenXtrue_dist(n, nrhs, xtrue, ldx);
-	dFillRHS_dist(trans, nrhs, xtrue, ldx, &A, b, ldb);
+
+  	if ( iam==0 ) {
+	    dGenXtrue_dist(n, nrhs, xtrue, ldx);
+	    dFillRHS_dist(trans, nrhs, xtrue, ldx, &A, b, ldb);
+
+            MPI_Bcast( xtrue, n*nrhs, MPI_DOUBLE, 0, grid1.comm );
+            MPI_Bcast( b, m*nrhs, MPI_DOUBLE, 0, grid1.comm );
+	} else {
+            MPI_Bcast( xtrue, n*nrhs, MPI_DOUBLE, 0, grid1.comm );
+            MPI_Bcast( b, m*nrhs, MPI_DOUBLE, 0, grid1.comm );
+	}
 
 	if ( !(berr = doubleMalloc_dist(nrhs)) )
 	    ABORT("Malloc fails for berr[].");
@@ -282,8 +291,16 @@ int main(int argc, char *argv[])
 	*trans = 'N';
 	ldx = n;
 	ldb = m;
-	dGenXtrue_dist(n, nrhs, xtrue, ldx);
-	dFillRHS_dist(trans, nrhs, xtrue, ldx, &A, b, ldb);
+
+        if ( iam==0 ) {
+	    dGenXtrue_dist(n, nrhs, xtrue, ldx);
+	    dFillRHS_dist(trans, nrhs, xtrue, ldx, &A, b, ldb);
+            MPI_Bcast( xtrue, n*nrhs, MPI_DOUBLE, 0, grid2.comm );
+            MPI_Bcast( b, m*nrhs, MPI_DOUBLE, 0, grid2.comm );
+	} else {
+            MPI_Bcast( xtrue, n*nrhs, MPI_DOUBLE, 0, grid2.comm );
+            MPI_Bcast( b, m*nrhs, MPI_DOUBLE, 0, grid2.comm );
+	}
 
 	if ( !(berr = doubleMalloc_dist(nrhs)) )
 	    ABORT("Malloc fails for berr[].");

diff --git a/EXAMPLE/pddrive_ABglobal.c b/EXAMPLE/pddrive_ABglobal.c
@@ -162,8 +162,17 @@ int main(int argc, char *argv[])
     *trans = 'N';
     ldx = n;
     ldb = m;
-    dGenXtrue_dist(n, nrhs, xtrue, ldx);
-    dFillRHS_dist(trans, nrhs, xtrue, ldx, &A, b, ldb);
+
+    if ( iam==0 ) {
+        dGenXtrue_dist(n, nrhs, xtrue, ldx);
+        dFillRHS_dist(trans, nrhs, xtrue, ldx, &A, b, ldb);
+
+        MPI_Bcast( xtrue, n*nrhs, MPI_DOUBLE, 0, grid.comm );
+        MPI_Bcast( b, m*nrhs, MPI_DOUBLE, 0, grid.comm );
+    } else {
+        MPI_Bcast( xtrue, n*nrhs, MPI_DOUBLE, 0, grid.comm );
+        MPI_Bcast( b, m*nrhs, MPI_DOUBLE, 0, grid.comm );
+    }
 
     if ( !(berr = doubleMalloc_dist(nrhs)) )
 	ABORT("Malloc fails for berr[].");

diff --git a/EXAMPLE/psdrive.c b/EXAMPLE/psdrive.c
@@ -162,22 +162,22 @@ int main(int argc, char *argv[])
             ttemp = getenv ("SUPERLU_BIND_MPI_GPU");
 
             if (ttemp) {
-	        int devs, rank;
-	        MPI_Comm_rank(MPI_COMM_WORLD, &rank); // MPI_COMM_WORLD needs to be used here instead of SubComm
+    	        int devs, rank;
+    	        MPI_Comm_rank(MPI_COMM_WORLD, &rank); // MPI_COMM_WORLD needs to be used here instead of SubComm
 	        gpuGetDeviceCount(&devs);  // Returns the number of compute-capable devices
 	        gpuSetDevice(rank % devs); // Set device to be used for GPU executions
             }
 
-            // This is to initialize GPU, which can be costly.
-            double t1 = SuperLU_timer_();
+            // This is to initialize GPU, which can be costly. 
+            double t1 = SuperLU_timer_();                       
             gpuFree(0);
-            double t2 = SuperLU_timer_();
+            double t2 = SuperLU_timer_();    
             if(!myrank)printf("first gpufree time: %7.4f\n",t2-t1);
-            gpublasHandle_t hb;
+            gpublasHandle_t hb;           
             gpublasCreate(&hb);
             if(!myrank)printf("first blas create time: %7.4f\n",SuperLU_timer_()-t2);
             gpublasDestroy(hb);
-	}
+        }
 #endif
         // printf("grid.iam %5d, myrank %5d\n",grid.iam,myrank);
         // fflush(stdout);
@@ -192,11 +192,11 @@ int main(int argc, char *argv[])
         int superlu_acc_offload = get_acc_offload();
         if (superlu_acc_offload) {
             MPI_Comm_rank(MPI_COMM_WORLD, &myrank);
-            double t1 = SuperLU_timer_();
+            double t1 = SuperLU_timer_();                       
             gpuFree(0);
-            double t2 = SuperLU_timer_();
+            double t2 = SuperLU_timer_();    
             if(!myrank)printf("first gpufree time: %7.4f\n",t2-t1);
-            gpublasHandle_t hb;
+            gpublasHandle_t hb;           
             gpublasCreate(&hb);
             if(!myrank)printf("first blas create time: %7.4f\n",SuperLU_timer_()-t2);
             gpublasDestroy(hb);
@@ -359,7 +359,7 @@ int main(int argc, char *argv[])
         MPI_Allreduce(MPI_IN_PLACE, result_min, 2, MPI_FLOAT,MPI_MIN, MPI_COMM_WORLD);
         MPI_Allreduce(MPI_IN_PLACE, result_max, 2, MPI_FLOAT,MPI_MAX, MPI_COMM_WORLD);
         if (!myrank) {
-            printf("returning data:\n");
+            printf("Batch solves returning data:\n");
             printf("    Factor time over all grids.  Min: %8.4f Max: %8.4f\n",result_min[0], result_max[0]);
                         printf("    Solve time over all grids.  Min: %8.4f Max: %8.4f\n",result_min[1], result_max[1]);
             printf("**************************************************\n");

diff --git a/EXAMPLE/psdrive1_ABglobal.c b/EXAMPLE/psdrive1_ABglobal.c
@@ -159,8 +159,18 @@ int main(int argc, char *argv[])
     *trans = 'N';
     ldx = n;
     ldb = m;
-    sGenXtrue_dist(n, nrhs, xtrue, ldx);
-    sFillRHS_dist(trans, nrhs, xtrue, ldx, &A, b, ldb);
+
+    if ( iam==0 ) {
+        sGenXtrue_dist(n, nrhs, xtrue, ldx);
+        sFillRHS_dist(trans, nrhs, xtrue, ldx, &A, b, ldb);
+
+        MPI_Bcast( xtrue, n*nrhs, MPI_FLOAT, 0, grid.comm );
+        MPI_Bcast( b, m*nrhs, MPI_FLOAT, 0, grid.comm );
+    } else {
+        MPI_Bcast( xtrue, n*nrhs, MPI_FLOAT, 0, grid.comm );
+        MPI_Bcast( b, m*nrhs, MPI_FLOAT, 0, grid.comm );
+    }
+
     for (j = 0; j < nrhs; ++j)
 	for (i = 0; i < m; ++i) b1[i+j*ldb] = b[i+j*ldb];
 

diff --git a/EXAMPLE/psdrive2_ABglobal.c b/EXAMPLE/psdrive2_ABglobal.c
@@ -157,8 +157,17 @@ int main(int argc, char *argv[])
     *trans = 'N';
     ldx = n;
     ldb = m;
-    sGenXtrue_dist(n, nrhs, xtrue, ldx);
-    sFillRHS_dist(trans, nrhs, xtrue, ldx, &A, b, ldb);
+
+    if ( iam==0 ) {
+        sGenXtrue_dist(n, nrhs, xtrue, ldx);
+        sFillRHS_dist(trans, nrhs, xtrue, ldx, &A, b, ldb);
+
+        MPI_Bcast( xtrue, n*nrhs, MPI_FLOAT, 0, grid.comm );
+        MPI_Bcast( b, m*nrhs, MPI_FLOAT, 0, grid.comm );
+    } else {
+        MPI_Bcast( xtrue, n*nrhs, MPI_FLOAT, 0, grid.comm );
+        MPI_Bcast( b, m*nrhs, MPI_FLOAT, 0, grid.comm );
+    }
 
     /* Save a copy of the right-hand side. */  
     if ( !(b1 = floatMalloc_dist(m * nrhs)) ) ABORT("Malloc fails for b1[]");

diff --git a/EXAMPLE/psdrive3_ABglobal.c b/EXAMPLE/psdrive3_ABglobal.c
@@ -163,8 +163,17 @@ int main(int argc, char *argv[])
     *trans = 'N';
     ldx = n;
     ldb = m;
-    sGenXtrue_dist(n, nrhs, xtrue, ldx);
-    sFillRHS_dist(trans, nrhs, xtrue, ldx, &A, b, ldb);
+
+    if ( iam==0 ) {
+        sGenXtrue_dist(n, nrhs, xtrue, ldx);
+        sFillRHS_dist(trans, nrhs, xtrue, ldx, &A, b, ldb);
+
+        MPI_Bcast( xtrue, n*nrhs, MPI_FLOAT, 0, grid.comm );
+        MPI_Bcast( b, m*nrhs, MPI_FLOAT, 0, grid.comm );
+    } else {
+        MPI_Bcast( xtrue, n*nrhs, MPI_FLOAT, 0, grid.comm );
+        MPI_Bcast( b, m*nrhs, MPI_FLOAT, 0, grid.comm );
+    }
 
     /* Save a copy of the right-hand side. */  
     if ( !(b1 = floatMalloc_dist(m * nrhs)) ) ABORT("Malloc fails for b1[]");

diff --git a/EXAMPLE/psdrive4_ABglobal.c b/EXAMPLE/psdrive4_ABglobal.c
@@ -177,8 +177,17 @@ int main(int argc, char *argv[])
 	*trans = 'N';
 	ldx = n;
 	ldb = m;
-	sGenXtrue_dist(n, nrhs, xtrue, ldx);
-	sFillRHS_dist(trans, nrhs, xtrue, ldx, &A, b, ldb);
+
+  	if ( iam==0 ) {
+	    sGenXtrue_dist(n, nrhs, xtrue, ldx);
+	    sFillRHS_dist(trans, nrhs, xtrue, ldx, &A, b, ldb);
+
+            MPI_Bcast( xtrue, n*nrhs, MPI_FLOAT, 0, grid1.comm );
+            MPI_Bcast( b, m*nrhs, MPI_FLOAT, 0, grid1.comm );
+	} else {
+            MPI_Bcast( xtrue, n*nrhs, MPI_FLOAT, 0, grid1.comm );
+            MPI_Bcast( b, m*nrhs, MPI_FLOAT, 0, grid1.comm );
+	}
 
 	if ( !(berr = floatMalloc_dist(nrhs)) )
 	    ABORT("Malloc fails for berr[].");
@@ -282,8 +291,16 @@ int main(int argc, char *argv[])
 	*trans = 'N';
 	ldx = n;
 	ldb = m;
-	sGenXtrue_dist(n, nrhs, xtrue, ldx);
-	sFillRHS_dist(trans, nrhs, xtrue, ldx, &A, b, ldb);
+
+        if ( iam==0 ) {
+	    sGenXtrue_dist(n, nrhs, xtrue, ldx);
+	    sFillRHS_dist(trans, nrhs, xtrue, ldx, &A, b, ldb);
+            MPI_Bcast( xtrue, n*nrhs, MPI_FLOAT, 0, grid2.comm );
+            MPI_Bcast( b, m*nrhs, MPI_FLOAT, 0, grid2.comm );
+	} else {
+            MPI_Bcast( xtrue, n*nrhs, MPI_FLOAT, 0, grid2.comm );
+            MPI_Bcast( b, m*nrhs, MPI_FLOAT, 0, grid2.comm );
+	}
 
 	if ( !(berr = floatMalloc_dist(nrhs)) )
 	    ABORT("Malloc fails for berr[].");

diff --git a/EXAMPLE/psdrive_ABglobal.c b/EXAMPLE/psdrive_ABglobal.c
@@ -159,8 +159,17 @@ int main(int argc, char *argv[])
     *trans = 'N';
     ldx = n;
     ldb = m;
-    sGenXtrue_dist(n, nrhs, xtrue, ldx);
-    sFillRHS_dist(trans, nrhs, xtrue, ldx, &A, b, ldb);
+
+    if ( iam==0 ) {
+        sGenXtrue_dist(n, nrhs, xtrue, ldx);
+        sFillRHS_dist(trans, nrhs, xtrue, ldx, &A, b, ldb);
+
+        MPI_Bcast( xtrue, n*nrhs, MPI_FLOAT, 0, grid.comm );
+        MPI_Bcast( b, m*nrhs, MPI_FLOAT, 0, grid.comm );
+    } else {
+        MPI_Bcast( xtrue, n*nrhs, MPI_FLOAT, 0, grid.comm );
+        MPI_Bcast( b, m*nrhs, MPI_FLOAT, 0, grid.comm );
+    }
 
     if ( !(berr = floatMalloc_dist(nrhs)) )
 	ABORT("Malloc fails for berr[].");

diff --git a/EXAMPLE/pzdrive.c b/EXAMPLE/pzdrive.c
@@ -156,27 +156,27 @@ int main(int argc, char *argv[])
 #ifdef GPU_ACC
         int superlu_acc_offload = get_acc_offload();
         if (superlu_acc_offload) {
-	    /* Binding each MPI to a GPU device */
-	    char *ttemp;
-	    ttemp = getenv ("SUPERLU_BIND_MPI_GPU");
-
-	    if (ttemp) {
-		int devs, rank;
-		MPI_Comm_rank(MPI_COMM_WORLD, &rank); // MPI_COMM_WORLD needs to be used here instead of SubComm
-		gpuGetDeviceCount(&devs);  // Returns the number of compute-capable devices
-		gpuSetDevice(rank % devs); // Set device to be used for GPU executions
-	    }
+            /* Binding each MPI to a GPU device */
+            char *ttemp;
+            ttemp = getenv ("SUPERLU_BIND_MPI_GPU");
+
+            if (ttemp) {
+    	        int devs, rank;
+    	        MPI_Comm_rank(MPI_COMM_WORLD, &rank); // MPI_COMM_WORLD needs to be used here instead of SubComm
+	        gpuGetDeviceCount(&devs);  // Returns the number of compute-capable devices
+	        gpuSetDevice(rank % devs); // Set device to be used for GPU executions
+            }
 
-	    // This is to initialize GPU, which can be costly.
-	    double t1 = SuperLU_timer_();
-	    gpuFree(0);
-	    double t2 = SuperLU_timer_();
-	    if(!myrank)printf("first gpufree time: %7.4f\n",t2-t1);
-	    gpublasHandle_t hb;
-	    gpublasCreate(&hb);
-	    if(!myrank)printf("first blas create time: %7.4f\n",SuperLU_timer_()-t2);
-	    gpublasDestroy(hb);
-	}
+            // This is to initialize GPU, which can be costly. 
+            double t1 = SuperLU_timer_();                       
+            gpuFree(0);
+            double t2 = SuperLU_timer_();    
+            if(!myrank)printf("first gpufree time: %7.4f\n",t2-t1);
+            gpublasHandle_t hb;           
+            gpublasCreate(&hb);
+            if(!myrank)printf("first blas create time: %7.4f\n",SuperLU_timer_()-t2);
+            gpublasDestroy(hb);
+        }
 #endif
         // printf("grid.iam %5d, myrank %5d\n",grid.iam,myrank);
         // fflush(stdout);
@@ -190,15 +190,15 @@ int main(int argc, char *argv[])
 #ifdef GPU_ACC
         int superlu_acc_offload = get_acc_offload();
         if (superlu_acc_offload) {
-	    MPI_Comm_rank(MPI_COMM_WORLD, &myrank);
-	    double t1 = SuperLU_timer_();
-	    gpuFree(0);
-	    double t2 = SuperLU_timer_();
-	    if(!myrank)printf("first gpufree time: %7.4f\n",t2-t1);
-	    gpublasHandle_t hb;
-	    gpublasCreate(&hb);
-	    if(!myrank)printf("first blas create time: %7.4f\n",SuperLU_timer_()-t2);
-	    gpublasDestroy(hb);
+            MPI_Comm_rank(MPI_COMM_WORLD, &myrank);
+            double t1 = SuperLU_timer_();                       
+            gpuFree(0);
+            double t2 = SuperLU_timer_();    
+            if(!myrank)printf("first gpufree time: %7.4f\n",t2-t1);
+            gpublasHandle_t hb;           
+            gpublasCreate(&hb);
+            if(!myrank)printf("first blas create time: %7.4f\n",SuperLU_timer_()-t2);
+            gpublasDestroy(hb);
 	}
 #endif
     }
@@ -357,7 +357,7 @@ int main(int argc, char *argv[])
         MPI_Allreduce(MPI_IN_PLACE, result_min, 2, MPI_FLOAT,MPI_MIN, MPI_COMM_WORLD);
         MPI_Allreduce(MPI_IN_PLACE, result_max, 2, MPI_FLOAT,MPI_MAX, MPI_COMM_WORLD);
         if (!myrank) {
-            printf("returning data:\n");
+            printf("Batch solves returning data:\n");
             printf("    Factor time over all grids.  Min: %8.4f Max: %8.4f\n",result_min[0], result_max[0]);
                         printf("    Solve time over all grids.  Min: %8.4f Max: %8.4f\n",result_min[1], result_max[1]);
             printf("**************************************************\n");

diff --git a/EXAMPLE/pzdrive1_ABglobal.c b/EXAMPLE/pzdrive1_ABglobal.c
@@ -158,8 +158,18 @@ int main(int argc, char *argv[])
     *trans = 'N';
     ldx = n;
     ldb = m;
-    zGenXtrue_dist(n, nrhs, xtrue, ldx);
-    zFillRHS_dist(trans, nrhs, xtrue, ldx, &A, b, ldb);
+
+    if ( iam==0 ) {
+        zGenXtrue_dist(n, nrhs, xtrue, ldx);
+        zFillRHS_dist(trans, nrhs, xtrue, ldx, &A, b, ldb);
+
+        MPI_Bcast( xtrue, n*nrhs, SuperLU_MPI_DOUBLE_COMPLEX, 0, grid.comm );
+        MPI_Bcast( b, m*nrhs, SuperLU_MPI_DOUBLE_COMPLEX, 0, grid.comm );
+    } else {
+        MPI_Bcast( xtrue, n*nrhs, SuperLU_MPI_DOUBLE_COMPLEX, 0, grid.comm );
+        MPI_Bcast( b, m*nrhs, SuperLU_MPI_DOUBLE_COMPLEX, 0, grid.comm );
+    }
+
     for (j = 0; j < nrhs; ++j)
 	for (i = 0; i < m; ++i) b1[i+j*ldb] = b[i+j*ldb];