Changeset 23641

Timestamp:

01/17/19 12:00:34 (6 years ago)

Author:

Mathieu Morlighem

Message:

CHG: speeding up allocation without MPI_Allreduce

Location:

issm/trunk-jpl/src/c

Files:

• classes/Node.cpp (modified) (1 diff)
• classes/Node.h (modified) (1 diff)
• classes/Nodes.cpp (modified) (1 diff)
• classes/Nodes.h (modified) (1 diff)
• modules/AllocateSystemMatricesx/AllocateSystemMatricesx.cpp (modified) (7 diffs)

issm/trunk-jpl/src/c/classes/Node.cpp

@@ -623 +623 @@
 }
 /*}}}*/
+int  Node::Pid(void){/*{{{*/
+        return this->pid; 
+}
+/*}}}*/
 
 /*Node numerics:*/

issm/trunk-jpl/src/c/classes/Node.h

@@ -108 +108 @@
                 void  ShowMasterDofs(int* truerows,int setenum);
                 int   Sid(void); 
+                int   Pid(void); 
                 void  UpdateCloneDofs(int* alltruerows,int setenum);
                 void  VecMerge(Vector<IssmDouble>* ug,IssmDouble* local_uf,int* indices_uf,IssmDouble* local_ys,int* indices_ys);

issm/trunk-jpl/src/c/classes/Nodes.cpp

@@ -360 +360 @@
 }
 /*}}}*/
+int   Nodes::NumberOfNodesLocal(void){/*{{{*/
+
+        return this->numberofmasters_local;
+}
+/*}}}*/
 bool  Nodes::RequiresDofReindexing(void){/*{{{*/
 

issm/trunk-jpl/src/c/classes/Nodes.h

@@ -44 +44 @@
                 int   NumberOfDofsLocalAll(int setenum);
                 int   NumberOfNodes(void);
+                int   NumberOfNodesLocal(void);
                 bool  RequiresDofReindexing(void);
 };

issm/trunk-jpl/src/c/modules/AllocateSystemMatricesx/AllocateSystemMatricesx.cpp

@@ -108 +108 @@
         /*Get vector size and number of nodes*/
         int numnodes            = femmodel->nodes->NumberOfNodes();
+        int localmasters        = femmodel->nodes->NumberOfNodesLocal();
         int localnumnodes       = femmodel->nodes->Size();
         int numberofdofspernode = femmodel->nodes->MaxNumDofs(GsetEnum);
@@ -171 +172 @@
         bool *flags                  = xNew<bool>(localnumnodes);
         int  *flagsindices           = xNew<int>(localnumnodes);
-        int  *d_connectivity         = xNewZeroInit<int>(numnodes);
-        int  *o_connectivity         = xNewZeroInit<int>(numnodes);
-        int  *connectivity_clone     = xNewZeroInit<int>(numnodes);
+        int  *d_connectivity         = xNewZeroInit<int>(localnumnodes);
+        int  *o_connectivity         = xNewZeroInit<int>(localnumnodes);
         int  *all_connectivity_clone = xNewZeroInit<int>(numnodes);
+
+        Vector<IssmDouble> *connectivity_clone= new Vector<IssmDouble>(localmasters,numnodes);
 
         /*Resetting flags to false at eahc iteration takes a lot of time, so we keep track of the flags
@@ -209 +211 @@
                         element->SetwiseNodeConnectivity(&d_nz,&o_nz,node,flags,flagsindices,set1enum,set2enum);
                         if(node->IsClone()){
-                                connectivity_clone[node->Sid()]+=d_nz+o_nz;
-                        }
-                        else{
-                                d_connectivity[node->Sid()]+=d_nz;
-                                o_connectivity[node->Sid()]+=o_nz;
+                                connectivity_clone->SetValue(node->Pid(),d_nz+o_nz,ADD_VAL);
+                        }
+                        else{
+                                d_connectivity[node->Lid()]+=d_nz;
+                                o_connectivity[node->Lid()]+=o_nz;
                         }
                 }
@@ -221 +223 @@
                         load->SetwiseNodeConnectivity(&d_nz,&o_nz,node,flags,flagsindices,set1enum,set2enum);
                         if(node->IsClone()){
-                                connectivity_clone[node->Sid()]+=d_nz+o_nz;
-                        }
-                        else{
-                                d_connectivity[node->Sid()]+=d_nz;
-                                o_connectivity[node->Sid()]+=o_nz;
+                                connectivity_clone->SetValue(node->Pid(),d_nz+o_nz,ADD_VAL);
+                        }
+                        else{
+                                d_connectivity[node->Lid()]+=d_nz;
+                                o_connectivity[node->Lid()]+=o_nz;
                         }
                 }
@@ -239 +241 @@
 
         /*sum over all cpus*/
-        ISSM_MPI_Allreduce((void*)connectivity_clone,(void*)all_connectivity_clone,numnodes,ISSM_MPI_INT,ISSM_MPI_SUM,IssmComm::GetComm());
-        xDelete<int>(connectivity_clone);
+        connectivity_clone->Assemble();
+        IssmDouble* serial_connectivity_clone=NULL;
+        femmodel->GetLocalVectorWithClonesVertices(&serial_connectivity_clone,connectivity_clone);
+        delete connectivity_clone;
 
         if(set1enum==FsetEnum){
@@ -251 +255 @@
                                 for(j=0;j<node->fsize;j++){
                                         _assert_(count<m);
-                                        d_nnz[count]=numberofdofspernode*(d_connectivity[node->Sid()] + all_connectivity_clone[node->Sid()]);
-                                        o_nnz[count]=numberofdofspernode*(o_connectivity[node->Sid()] + all_connectivity_clone[node->Sid()]);
+                                        d_nnz[count]=numberofdofspernode*(d_connectivity[node->Lid()] + reCast<int>(serial_connectivity_clone[node->Lid()]));
+                                        o_nnz[count]=numberofdofspernode*(o_connectivity[node->Lid()] + reCast<int>(serial_connectivity_clone[node->Lid()]));
                                         if(d_nnz[count]>n)   d_nnz[count]=n;
                                         if(o_nnz[count]>N-n) o_nnz[count]=N-n;
@@ -266 +270 @@
         xDelete<int>(d_connectivity);
         xDelete<int>(o_connectivity);
-        xDelete<int>(all_connectivity_clone);
+        xDelete<IssmDouble>(serial_connectivity_clone);
 
         /*Allocate ouptput pointer*/