Changeset 27382

Timestamp:

11/10/22 17:21:17 (2 years ago)

Author:

Mathieu Morlighem

Message:

CHG: more deadlocking fix

Location:

issm/trunk-jpl/src/c/classes

Files:

• FemModel.cpp (modified) (1 diff)
• Nodes.cpp (modified) (4 diffs)

issm/trunk-jpl/src/c/classes/FemModel.cpp

@@ -1589 +1589 @@
         /*Now send and receive vector for nodes on partition edge*/
         IssmDouble **send_buffers = xNewZeroInit<IssmDouble*>(num_procs);
-        IssmDouble  *recv_buffer  = xNewZeroInit<IssmDouble>(this->nodes->Size());
+        IssmDouble  *recv_buffer  = xNewZeroInit<IssmDouble>(this->nodes->Size(),"t"); //only one alloc, "t" is required by adolc
         ISSM_MPI_Request  *send_requests = xNew<ISSM_MPI_Request>(num_procs);
         for (int rank = 0;rank<num_procs;rank++) send_requests[rank] = ISSM_MPI_REQUEST_NULL;

issm/trunk-jpl/src/c/classes/Nodes.cpp

@@ -272 +272 @@
 
         /* Share pids of masters and update pids of clones*/
-        int* truepids = xNew<int>(this->Size()); //only one alloc
+        int **send_truepids = xNewZeroInit<int*>(num_procs);
+        int  *recv_truepids = xNewZeroInit<int>(this->Size());
+        ISSM_MPI_Request* send_requests = xNew<ISSM_MPI_Request>(num_procs);
+        for(int rank=0;rank<num_procs;rank++) send_requests[rank] = MPI_REQUEST_NULL;
         for(int rank=0;rank<num_procs;rank++){
                 if(this->common_send[rank]){
                         int  numids = this->common_send[rank];
+                        send_truepids[rank] = xNew<int>(numids);
                         for(int i=0;i<numids;i++){
                                 Node* node=xDynamicCast<Node*>(this->GetObjectByOffset(this->common_send_ids[rank][i]));
-                                truepids[i] = node->pid;
-                        }
-                        ISSM_MPI_Send(truepids,numids,ISSM_MPI_INT,rank,0,IssmComm::GetComm());
+                                send_truepids[rank][i] = node->pid;
+                        }
+                        ISSM_MPI_Isend(send_truepids[rank],numids,ISSM_MPI_INT,rank,0,IssmComm::GetComm(),&send_requests[rank]);
                 }
         }
@@ -286 +290 @@
                 if(this->common_recv[rank]){
                         int  numids = this->common_recv[rank];
-                        ISSM_MPI_Recv(truepids,numids,ISSM_MPI_INT,rank,0,IssmComm::GetComm(),&status);
+                        ISSM_MPI_Recv(recv_truepids,numids,ISSM_MPI_INT,rank,0,IssmComm::GetComm(),&status);
                         for(int i=0;i<numids;i++){
                                 Node* node=xDynamicCast<Node*>(this->GetObjectByOffset(this->common_recv_ids[rank][i]));
-                                node->pid = truepids[i];
-                        }
-                }
-        }
-        xDelete<int>(truepids);
+                                node->pid = recv_truepids[i];
+                        }
+                }
+        }
+        xDelete<int>(recv_truepids);
+        for(int rank=0;rank<num_procs;rank++){
+                if(rank!=my_rank) ISSM_MPI_Wait(&send_requests[rank],&status);
+                xDelete<int>(send_truepids[rank]);
+        }
+        xDelete<int*>(send_truepids);
+        xDelete<ISSM_MPI_Request>(send_requests);
 
         /*4. Distribute G dofs once for all*/
@@ -494 +504 @@
 
         /*Now send and receive ug for nodes on partition edge*/
-        #ifdef _HAVE_AD_
-        IssmDouble* buffer = xNew<IssmDouble>(this->Size()*maxdofspernode,"t"); //only one alloc, "t" is required by adolc
-        #else
-        IssmDouble* buffer = xNew<IssmDouble>(this->Size()*maxdofspernode);
-        #endif
+        IssmDouble **send_buffers = xNewZeroInit<IssmDouble*>(num_procs);
+        IssmDouble  *recv_buffer  = xNewZeroInit<IssmDouble>(this->Size()*maxdofspernode,"t"); 
+        ISSM_MPI_Request  *send_requests = xNew<ISSM_MPI_Request>(num_procs);
+        for (int rank = 0;rank<num_procs;rank++) send_requests[rank] = ISSM_MPI_REQUEST_NULL;
+
         for(int rank=0;rank<num_procs;rank++){
                 if(this->common_send[rank]){
                         int  numids = this->common_send[rank];
+                        send_buffers[rank] = xNew<IssmDouble>(numids*maxdofspernode,"t"); //"t" is required by adolc
                         for(int i=0;i<numids;i++){
                                 int   master_lid = this->common_send_ids[rank][i];
                                 Node* node=xDynamicCast<Node*>(this->GetObjectByOffset(master_lid));
                                 _assert_(!node->IsClone());
-                                for(int j=0;j<node->gsize;j++) buffer[i*maxdofspernode+j]=local_ug[node->gdoflist_local[j]];
-                        }
-                        ISSM_MPI_Send(buffer,numids*maxdofspernode,ISSM_MPI_DOUBLE,rank,0,IssmComm::GetComm());
+                                for(int j=0;j<node->gsize;j++) send_buffers[rank][i*maxdofspernode+j]=local_ug[node->gdoflist_local[j]];
+                        }
+                        ISSM_MPI_Isend(send_buffers[rank],numids*maxdofspernode,ISSM_MPI_DOUBLE,rank,0,IssmComm::GetComm(),&send_requests[rank]);
                 }
         }
@@ -514 +525 @@
                 if(this->common_recv[rank]){
                         int  numids = this->common_recv[rank];
-                        ISSM_MPI_Recv(buffer,numids*maxdofspernode,ISSM_MPI_DOUBLE,rank,0,IssmComm::GetComm(),&status);
+                        ISSM_MPI_Recv(recv_buffer,numids*maxdofspernode,ISSM_MPI_DOUBLE,rank,0,IssmComm::GetComm(),&status);
                         for(int i=0;i<numids;i++){
                                 int   master_lid = this->common_recv_ids[rank][i];
                                 Node* node=xDynamicCast<Node*>(this->GetObjectByOffset(master_lid));
-                                for(int j=0;j<node->gsize;j++) local_ug[node->gdoflist_local[j]] = buffer[i*maxdofspernode+j];
-                        }
-                }
-        }
-        xDelete<IssmDouble>(buffer);
+                                for(int j=0;j<node->gsize;j++) local_ug[node->gdoflist_local[j]] = recv_buffer[i*maxdofspernode+j];
+                        }
+                }
+        }
+
+        xDelete<IssmDouble>(recv_buffer);
+        for(int rank=0;rank<num_procs;rank++){
+                if(rank!=my_rank) ISSM_MPI_Wait(&send_requests[rank],&status);
+                xDelete<IssmDouble>(send_buffers[rank]);
+        }
+        xDelete<IssmDouble*>(send_buffers);
+        xDelete<ISSM_MPI_Request>(send_requests);
 
         /*Assign output pointer*/