Changeset 23643

Timestamp:

01/17/19 15:04:23 (6 years ago)

Author:

Mathieu Morlighem

Message:

CHG: speed up output by not broadcasting to all cpus if only cpu 0 writes the output

Location:

issm/trunk-jpl/src/c

Files:

• classes/ExternalResults/GenericExternalResult.h (modified) (3 diffs)
• toolkits/issm/IssmAbsVec.h (modified) (1 diff)
• toolkits/issm/IssmMpiVec.h (modified) (1 diff)
• toolkits/issm/IssmSeqVec.h (modified) (1 diff)
• toolkits/issm/IssmVec.h (modified) (1 diff)
• toolkits/objects/Vector.h (modified) (1 diff)
• toolkits/petsc/objects/PetscVec.cpp (modified) (1 diff)
• toolkits/petsc/objects/PetscVec.h (modified) (1 diff)
• toolkits/petsc/patches/VecToMPISerial.cpp (modified) (3 diffs)
• toolkits/petsc/patches/petscpatches.h (modified) (1 diff)

issm/trunk-jpl/src/c/classes/ExternalResults/GenericExternalResult.h

@@ -719 +719 @@
                 }
 
-                /*Serialize vector*/
-                IssmPDouble* serialvalues = this->value->ToMPISerial();
-                this->value->GetSize(&rows);
+                /*Serialize vector on cpu0*/
+                IssmPDouble* serialvalues = this->value->ToMPISerial0();
 
                 if(IssmComm::GetRank()==0){
+                        this->value->GetSize(&rows);
+
                         /*First write name: */
                         length=(strlen(this->result_name)+1)*sizeof(char);
@@ -770 +771 @@
                 }
 
-                /*Serialize vector*/
-                serialvalues = this->value->ToMPISerial();
-                this->value->GetSize(&rows);
-
-                pserialvalues=xNew<IssmPDouble>(rows);
-                for(i=0;i<rows;i++)pserialvalues[i]=reCast<IssmPDouble>(serialvalues[i]);
+                /*Serialize vector only on cpu0*/
+                serialvalues = this->value->ToMPISerial0();
 
                 if(IssmComm::GetRank()==0){
+
+                        /*Make it passive*/
+                        this->value->GetSize(&rows);
+                        pserialvalues=xNew<IssmPDouble>(rows);
+                        for(i=0;i<rows;i++)pserialvalues[i]=reCast<IssmPDouble>(serialvalues[i]);
+
                         /*First write name: */
                         length=(strlen(this->result_name)+1)*sizeof(char);
@@ -794 +797 @@
                         fwrite(&cols,sizeof(int),1,fid);
                         fwrite(pserialvalues,cols*rows*sizeof(IssmPDouble),1,fid);
+
+                        /*Clean up*/
+                        xDelete<IssmPDouble>(pserialvalues);
                 }
 
                 /*Clean up*/
-                xDelete<IssmPDouble>(pserialvalues);
                 xDelete<IssmDouble>(serialvalues);
-
         }
         /*}}}*/

issm/trunk-jpl/src/c/toolkits/issm/IssmAbsVec.h

@@ -43 +43 @@
                 virtual void AYPX(IssmAbsVec* X, doubletype a)=0;
                 virtual doubletype* ToMPISerial(void)=0;
+                virtual doubletype* ToMPISerial0(void)=0;
                 virtual void Shift(doubletype shift)=0;
                 virtual void Copy(IssmAbsVec* to)=0;

issm/trunk-jpl/src/c/toolkits/issm/IssmMpiVec.h

@@ -460 +460 @@
                         /*return: */
                         return buffer;
+
+                }
+                /*}}}*/
+                doubletype* ToMPISerial0(void){/*{{{*/
+
+                        /*FIXME: Should not broadcast to every cpu*/
+                        return this->ToMPISerial();
 
                 }

issm/trunk-jpl/src/c/toolkits/issm/IssmSeqVec.h

@@ -218 +218 @@
                 }
                 /*}}}*/
+                doubletype* ToMPISerial0(void){/*{{{*/
+
+                        return this->ToMPISerial();
+
+                }
+                /*}}}*/
                 void Shift(doubletype shift){/*{{{*/
 

issm/trunk-jpl/src/c/toolkits/issm/IssmVec.h

@@ -183 +183 @@
                 }
                 /*}}}*/
+                doubletype* ToMPISerial0(void){/*{{{*/
+                        return vector->ToMPISerial0();
+                }
+                /*}}}*/
                 void Shift(doubletype shift){/*{{{*/
                         vector->Shift(shift);

issm/trunk-jpl/src/c/toolkits/objects/Vector.h

@@ -291 +291 @@
                 }
                 /*}}}*/
+                doubletype* ToMPISerial0(void){/*{{{*/
+
+                        doubletype* vec_serial=NULL;
+
+                        _assert_(this);
+                        if(type==PetscVecType){
+                                #ifdef _HAVE_PETSC_
+                                vec_serial=this->pvector->ToMPISerial0();
+                                #endif
+                        }
+                        else vec_serial=this->ivector->ToMPISerial0();
+
+                        return vec_serial;
+
+                }
+                /*}}}*/
                 void Shift(doubletype shift){_assert_(this);/*{{{*/
 

issm/trunk-jpl/src/c/toolkits/petsc/objects/PetscVec.cpp

@@ -191 +191 @@
 
         IssmDouble* vec_serial=NULL;
-        VecToMPISerial(&vec_serial, this->vector,IssmComm::GetComm());
+        VecToMPISerial(&vec_serial, this->vector,IssmComm::GetComm(),true);
+        return vec_serial;
+
+}
+/*}}}*/
+IssmDouble* PetscVec::ToMPISerial0(void){/*{{{*/
+
+        IssmDouble* vec_serial=NULL;
+        VecToMPISerial(&vec_serial, this->vector,IssmComm::GetComm(),false);
         return vec_serial;
 

issm/trunk-jpl/src/c/toolkits/petsc/objects/PetscVec.h

@@ -51 +51 @@
                 void        AYPX(PetscVec* X, IssmDouble a);
                 IssmDouble* ToMPISerial(void);
+                IssmDouble* ToMPISerial0(void);
                 void        Shift(IssmDouble shift);
                 void        Copy(PetscVec* to);

issm/trunk-jpl/src/c/toolkits/petsc/patches/VecToMPISerial.cpp

@@ -12 +12 @@
 #include "../../../shared/shared.h"
 
-int VecToMPISerial(double** pgathered_vector, Vec vector,ISSM_MPI_Comm comm){
+int VecToMPISerial(double** pgathered_vector, Vec vector,ISSM_MPI_Comm comm,bool broadcast){
 
         int i;
@@ -45 +45 @@
 
         /*Allocate gathered vector on all nodes .*/
-        gathered_vector=xNew<double>(vector_size);
+        if(broadcast || my_rank==0){ 
+                gathered_vector=xNew<double>(vector_size);
+        }
 
         /*Allocate local vectors*/
@@ -83 +85 @@
         }
 
-        /*Now, broadcast gathered_vector from node 0 to other nodes: */
-        ISSM_MPI_Bcast(gathered_vector,vector_size,ISSM_MPI_PDOUBLE,0,comm);
+        if(broadcast){
+                /*Now, broadcast gathered_vector from node 0 to other nodes: */
+                ISSM_MPI_Bcast(gathered_vector,vector_size,ISSM_MPI_PDOUBLE,0,comm);
+        }
 
         /*Assign output pointers: */

issm/trunk-jpl/src/c/toolkits/petsc/patches/petscpatches.h

@@ -22 +22 @@
 Mat NewMat(int M,int N,int connectivity,int numberofdofspernode, ISSM_MPI_Comm comm);
 
-int VecToMPISerial(double** pgathered_vector, Vec vector,ISSM_MPI_Comm comm);
+int VecToMPISerial(double** pgathered_vector, Vec vector,ISSM_MPI_Comm comm,bool broadcast=true);
 void MatFree(Mat* pmat);
 void ISFree(IS* pis);