Changeset 23600

Timestamp:

01/04/19 21:15:03 (6 years ago)

Author:

Mathieu Morlighem

Message:

NEW: new implementation of dof distribution, passing only clone information

Location:

issm/trunk-jpl/src/c

Files:

• classes/Node.cpp (modified) (4 diffs)
• classes/Node.h (modified) (1 diff)
• classes/Nodes.cpp (modified) (3 diffs)
• modules/NodesDofx/NodesDofx.cpp (modified) (1 diff)

issm/trunk-jpl/src/c/classes/Node.cpp

@@ -500 +500 @@
 }
 /*}}}*/
-void  Node::SetApproximation(int in_approximation){/*{{{*/
+void Node::SetApproximation(int in_approximation){/*{{{*/
 
         this->approximation = in_approximation;
@@ -749 +749 @@
 }
 /*}}}*/
-void Node::ShowTrueDofs(int* truedofs, int ncols,int setenum){/*{{{*/
-
-        int j;
-
-        /*Are we a clone? : */
-        if(indexing.clone) return;
+void Node::ShowTrueDofs(int* truedofs,int setenum){/*{{{*/
+
+        _assert_(!this->indexing.clone);
 
         /*Ok, we are not a clone, just plug our dofs into truedofs: */
         switch(setenum){
                 case GsetEnum:
-                        for(j=0;j<this->indexing.gsize;j++) truedofs[ncols*sid+j]=indexing.gdoflist[j];
+                        for(int j=0;j<this->indexing.gsize;j++) truedofs[j]=indexing.gdoflist[j];
                         break;
                 case FsetEnum:
-                        for(j=0;j<this->indexing.fsize;j++) truedofs[ncols*sid+j]=indexing.fdoflist[j];
+                        for(int j=0;j<this->indexing.fsize;j++) truedofs[j]=indexing.fdoflist[j];
                         break;
                 case SsetEnum:
-                        for(j=0;j<this->indexing.ssize;j++) truedofs[ncols*sid+j]=indexing.sdoflist[j];
+                        for(int j=0;j<this->indexing.ssize;j++) truedofs[j]=indexing.sdoflist[j];
                         break;
                 default:
@@ -773 +770 @@
 }
 /*}}}*/
-void Node::UpdateCloneDofs(int* alltruedofs,int ncols,int setenum){/*{{{*/
-
-        int j;
-
-        /*If we are not a clone, don't update, we already have dofs!: */
-        if(!indexing.clone)return;
+void Node::UpdateCloneDofs(int* alltruedofs,int setenum){/*{{{*/
+
+        _assert_(this->indexing.clone);
 
         /*Ok, we are a clone node, but we did not create the dofs for this node.
@@ -784 +778 @@
         switch(setenum){
                 case GsetEnum:
-                        for(j=0;j<this->indexing.gsize;j++) indexing.gdoflist[j]=alltruedofs[ncols*sid+j];
+                        for(int j=0;j<this->indexing.gsize;j++) indexing.gdoflist[j]=alltruedofs[j];
                         break;
                 case FsetEnum:
-                        for(j=0;j<this->indexing.fsize;j++) indexing.fdoflist[j]=alltruedofs[ncols*sid+j];
+                        for(int j=0;j<this->indexing.fsize;j++) indexing.fdoflist[j]=alltruedofs[j];
                         break;
                 case SsetEnum:
-                        for(j=0;j<this->indexing.ssize;j++) indexing.sdoflist[j]=alltruedofs[ncols*sid+j];
+                        for(int j=0;j<this->indexing.ssize;j++) indexing.sdoflist[j]=alltruedofs[j];
                         break;
                 default:

issm/trunk-jpl/src/c/classes/Node.h

@@ -78 +78 @@
                 bool  RequiresDofReindexing(void);
                 void  SetCurrentConfiguration(DataSet* nodes,Vertices* vertices);
-                void  ShowTrueDofs(int* truerows,int ncols,int setenum);
+                void  ShowTrueDofs(int* truerows,int setenum);
                 int   Sid(void); 
-                void  UpdateCloneDofs(int* alltruerows,int ncols,int setenum);
+                void  UpdateCloneDofs(int* alltruerows,int setenum);
                 void  VecMerge(Vector<IssmDouble>* ug, IssmDouble* vector_serial,int setenum);
                 void  VecReduce(Vector<IssmDouble>* vector, IssmDouble* ug_serial,int setnum);

issm/trunk-jpl/src/c/classes/Nodes.cpp

@@ -133 +133 @@
 void  Nodes::DistributeDofs(int setenum){/*{{{*/
 
-        /*some check: */
-        _assert_(setenum==GsetEnum || setenum==FsetEnum || setenum==SsetEnum);
-
-        int  i;
-        int  dofcount=0;
-        int  maxdofspernode=0;
-        int* alldofcount=NULL;
-        int* truedofs=NULL;
-        int* alltruedofs=NULL;
-        int  numnodes=0;
-
         /*recover my_rank:*/
+        ISSM_MPI_Status status;
         int my_rank   = IssmComm::GetRank();
         int num_procs = IssmComm::GetSize();
 
         /*Go through objects, and distribute dofs locally, from 0 to numberofdofsperobject*/
-        for(i=0;i<this->Size();i++){
+        int  dofcount=0;
+        for(int i=0;i<this->Size();i++){
                 Node* node=xDynamicCast<Node*>(this->GetObjectByOffset(i));
                 node->DistributeDofs(&dofcount,setenum);
@@ -158 +149 @@
          * cpus by the total last dofs of the previus cpu, starting from 0.
          * First: get number of dofs for each cpu*/
-        alldofcount=xNew<int>(num_procs);
+        int* alldofcount=xNew<int>(num_procs);
         ISSM_MPI_Gather(&dofcount,1,ISSM_MPI_INT,alldofcount,1,ISSM_MPI_INT,0,IssmComm::GetComm());
         ISSM_MPI_Bcast(alldofcount,num_procs,ISSM_MPI_INT,0,IssmComm::GetComm());
 
         /* Every cpu should start its own dof count at the end of the dofcount from cpu-1*/
-        dofcount=0;
-        for(i=0;i<my_rank;i++){
-                dofcount+=alldofcount[i];
-        }
-        for(i=0;i<this->Size();i++){
+        int offset=0;
+        for(int i=0;i<my_rank;i++) offset+=alldofcount[i];
+        xDelete<int>(alldofcount);
+
+        for(int i=0;i<this->Size();i++){
                 /*Check that this node corresponds to our analysis currently being carried out: */
                 Node* node=xDynamicCast<Node*>(this->GetObjectByOffset(i));
-                node->OffsetDofs(dofcount,setenum);
+                node->OffsetDofs(offset,setenum);
         }
 
@@ -176 +167 @@
          * object that is not a clone, tell them to show their dofs, so that later on, they can get picked 
          * up by their clones: */
-        maxdofspernode=this->MaxNumDofs(setenum);
-        numnodes=this->NumberOfNodes();
-        if(numnodes*maxdofspernode){
-                truedofs=   xNewZeroInit<int>(numnodes*maxdofspernode); //initialize to 0, so that we can pick up the max
-                alltruedofs=xNewZeroInit<int>(numnodes*maxdofspernode);
-        }
-
-        for(i=0;i<this->Size();i++){
-                Node* node=xDynamicCast<Node*>(this->GetObjectByOffset(i));
-                node->ShowTrueDofs(truedofs,maxdofspernode,setenum);//give maxdofspernode, column size, so that nodes can index into truedofs
-        }
-
-        ISSM_MPI_Allreduce((void*)truedofs,(void*)alltruedofs,numnodes*maxdofspernode,ISSM_MPI_INT,ISSM_MPI_MAX,IssmComm::GetComm());
-
-        /* Now every cpu knows the true dofs of everyone else that is not a clone*/
-        for(i=0;i<this->Size();i++){
-                Node* node=xDynamicCast<Node*>(this->GetObjectByOffset(i));
-                node->UpdateCloneDofs(alltruedofs,maxdofspernode,setenum);
-        }
+        int  maxdofspernode = this->MaxNumDofs(setenum);
+        int* truedofs       = xNew<int>(this->Size()*maxdofspernode); //only one alloc
+        for(int rank=0;rank<num_procs;rank++){
+                if(this->common_send[rank]){
+                        int  numids = this->common_send[rank];
+                        for(int i=0;i<numids;i++){
+                                Node* node=xDynamicCast<Node*>(this->GetObjectByOffset(this->common_send_ids[rank][i]));
+                                node->ShowTrueDofs(&truedofs[i*maxdofspernode+0],setenum);
+                        }
+                        ISSM_MPI_Send(truedofs,numids*maxdofspernode,ISSM_MPI_INT,rank,0,IssmComm::GetComm());
+                }
+        }
+        for(int rank=0;rank<num_procs;rank++){
+                if(this->common_recv[rank]){
+                        int  numids = this->common_recv[rank];
+                        ISSM_MPI_Recv(truedofs,numids*maxdofspernode,ISSM_MPI_INT,rank,0,IssmComm::GetComm(),&status);
+                        for(int i=0;i<numids;i++){
+                                Node* node=xDynamicCast<Node*>(this->GetObjectByOffset(this->common_recv_ids[rank][i]));
+                                node->UpdateCloneDofs(&truedofs[i*maxdofspernode+0],setenum);
+                        }
+                }
+        }
+        xDelete<int>(truedofs);
 
         /*Update indexingupdateflag*/
-        for(i=0;i<this->Size();i++){
+        for(int i=0;i<this->Size();i++){
                 Node* node=xDynamicCast<Node*>(this->GetObjectByOffset(i));
                 node->ReindexingDone();
         }
-
-        /* Free ressources: */
-        xDelete<int>(alldofcount);
-        xDelete<int>(truedofs);
-        xDelete<int>(alltruedofs);
 }
 /*}}}*/

issm/trunk-jpl/src/c/modules/NodesDofx/NodesDofx.cpp

@@ -24 +24 @@
         nodes->DistributeDofs(FsetEnum);
         nodes->DistributeDofs(SsetEnum);
-
 }