Changeset 23926

Timestamp:

05/20/19 16:33:50 (6 years ago)

Author:

Mathieu Morlighem

Message:

CHG: moving stuff to nodes, and added method to check dof consistency across partitions

Location:

issm/trunk-jpl/src/c/classes

Files:

• FemModel.cpp (modified) (1 diff)
• Nodes.cpp (modified) (3 diffs)
• Nodes.h (modified) (2 diffs)

issm/trunk-jpl/src/c/classes/FemModel.cpp

@@ -1308 +1308 @@
 void FemModel::GetLocalVectorWithClonesGset(IssmDouble** plocal_ug,Vector<IssmDouble> *ug){/*{{{*/
 
-        /*recover my_rank:*/
-        ISSM_MPI_Status status;
-        int my_rank   = IssmComm::GetRank();
-        int num_procs = IssmComm::GetSize();
-
-        /*retrieve node info*/
-        int glocalsize         = this->nodes->NumberOfDofsLocalAll(GsetEnum);
-        int glocalsize_masters = this->nodes->NumberOfDofsLocal(GsetEnum);
-        int maxdofspernode     = this->nodes->MaxNumDofs(GsetEnum);
-
-        /*Get local vector of ug*/
-        int        *indices_ug_masters = NULL;
-        IssmDouble *local_ug_masters   = NULL;
-        ug->GetLocalVector(&local_ug_masters,&indices_ug_masters);
-        _assert_(glocalsize_masters==indices_ug_masters[glocalsize_masters-1] - indices_ug_masters[0]+1);
-        xDelete<int>(indices_ug_masters);
-
-        /*Now, extend vectors to account for clones (make vectors longer, for clones at the end)*/
-        IssmDouble *local_ug  = xNew<IssmDouble>(glocalsize);
-        xMemCpy<IssmDouble>(local_ug,local_ug_masters,glocalsize_masters);
-        xDelete<IssmDouble>(local_ug_masters);
-
-        /*Now send and receive ug for nodes on partition edge*/
-        #ifdef _HAVE_AD_
-        IssmDouble* buffer = xNew<IssmDouble>(this->nodes->Size()*maxdofspernode,"t"); //only one alloc, "t" is required by adolc
-        #else
-        IssmDouble* buffer = xNew<IssmDouble>(this->nodes->Size()*maxdofspernode);
-        #endif
-        for(int rank=0;rank<num_procs;rank++){
-                if(this->nodes->common_send[rank]){
-                        int  numids = this->nodes->common_send[rank];
-                        for(int i=0;i<numids;i++){
-                                int   master_lid = this->nodes->common_send_ids[rank][i];
-                                Node* node=xDynamicCast<Node*>(this->nodes->GetObjectByOffset(master_lid));
-                                _assert_(!node->IsClone());
-                                for(int j=0;j<node->gsize;j++) buffer[i*maxdofspernode+j]=local_ug[node->gdoflist_local[j]];
-                        }
-                        ISSM_MPI_Send(buffer,numids*maxdofspernode,ISSM_MPI_DOUBLE,rank,0,IssmComm::GetComm());
-                }
-        }
-        for(int rank=0;rank<num_procs;rank++){
-                if(this->nodes->common_recv[rank]){
-                        int  numids = this->nodes->common_recv[rank];
-                        ISSM_MPI_Recv(buffer,numids*maxdofspernode,ISSM_MPI_DOUBLE,rank,0,IssmComm::GetComm(),&status);
-                        for(int i=0;i<numids;i++){
-                                int   master_lid = this->nodes->common_recv_ids[rank][i];
-                                Node* node=xDynamicCast<Node*>(this->nodes->GetObjectByOffset(master_lid));
-                                for(int j=0;j<node->gsize;j++) local_ug[node->gdoflist_local[j]] = buffer[i*maxdofspernode+j];
-                        }
-                }
-        }
-        xDelete<IssmDouble>(buffer);
-
-        /*Assign output pointer*/
-        *plocal_ug = local_ug;
+        this->nodes->GetLocalVectorWithClonesGset(plocal_ug,ug);
+
 }/*}}}*/
 void FemModel::GetLocalVectorWithClonesVertices(IssmDouble** plocal_vector,Vector<IssmDouble> *vector){/*{{{*/

issm/trunk-jpl/src/c/classes/Nodes.cpp

@@ -4 +4 @@
  */
 
-/*Headers: {{{*/
+/*Headers*/
 #ifdef HAVE_CONFIG_H
         #include <config.h>
@@ -10 +10 @@
 #error "Cannot compile with HAVE_CONFIG_H symbol! run configure first!"
 #endif
-
 #include "../shared/io/Comm/IssmComm.h"
 #include "./Nodes.h"
 #include "./Node.h"
-
 using namespace std;
-/*}}}*/
 
 /*Object constructors and destructor*/
@@ -391 +388 @@
 }
 /*}}}*/
+
+void  Nodes::CheckDofListAcrossPartitions(void){/*{{{*/
+
+        /*recover my_rank:*/
+        ISSM_MPI_Status status;
+        int my_rank   = IssmComm::GetRank();
+        int num_procs = IssmComm::GetSize();
+
+        /*Display message*/
+        if(VerboseModule()) _printf0_("   Checking degrees of freedom across partitions\n");
+
+        /*Allocate vector to check degrees of freedom*/
+        int gsize      = this->NumberOfDofs(GsetEnum);
+        int glocalsize = this->NumberOfDofsLocal(GsetEnum);
+        Vector<IssmDouble>* dofs_check=new Vector<IssmDouble>(glocalsize,gsize);
+
+        /*First, go over all nodes, and masters can write their f dof and -1 for s-set*/
+        for(int i=0;i<this->Size();i++){
+                Node* node=xDynamicCast<Node*>(this->GetObjectByOffset(i));
+
+                /*Skip if clone (will check later)*/
+                if(node->IsClone()) continue;
+
+                /*Write degree of freedom if active*/
+                int count = 0;
+                for(int j=0;j<node->gsize;j++){
+                        if(node->f_set[j]){
+                                if(node->s_set[j]) _error_("a degree of freedom is both in f and s set!");
+                                dofs_check->SetValue(node->gdoflist[j],reCast<IssmDouble>(node->fdoflist[count]),INS_VAL);
+                                count++;
+                        }
+                        else{
+                                if(node->s_set[j]==0) _error_("a degree of freedom is neither in f nor in s set!");
+                                dofs_check->SetValue(node->gdoflist[j],-1.,INS_VAL);
+                        }
+                }
+        }
+        dofs_check->Assemble();
+
+        /*Get local vector with both masters and slaves:*/
+        IssmDouble *local_dofs_check = NULL;
+        this->GetLocalVectorWithClonesGset(&local_dofs_check,dofs_check);
+        delete dofs_check;
+
+        /*Second, go over all nodes, and check that we still have what's expected...*/
+        for(int i=0;i<this->Size();i++){
+                Node* node=xDynamicCast<Node*>(this->GetObjectByOffset(i));
+
+                /*Write degree of freedom if active*/
+                int countg = 0;
+                int countf = 0;
+                int counts = 0;
+                for(int j=0;j<node->gsize;j++){
+                        int index = node->gdoflist_local[countg];
+                        if(node->f_set[j]){
+                                if(reCast<int>(local_dofs_check[index]) != node->fdoflist[countf]){
+                                        _error_("Dof #"<<j<<" of node sid "<<node->Sid()<<" not consistent: "<<local_dofs_check[index]<<"!="<<node->fdoflist[countf]);
+                                }
+                                countf++;
+                        }
+                        else{
+                                if(local_dofs_check[index] != -1.){
+                                        _error_("Dof #"<<j<<" of node sid "<<node->Sid()<<" not consistently in s set");
+                                }
+                                counts++;
+                        }
+                        countg++;
+                }
+        }
+
+        /*cleanup and return*/
+        xDelete<IssmDouble>(local_dofs_check);
+}/*}}}*/
+void  Nodes::GetLocalVectorWithClonesGset(IssmDouble** plocal_ug,Vector<IssmDouble> *ug){/*{{{*/
+
+        /*recover my_rank:*/
+        ISSM_MPI_Status status;
+        int my_rank   = IssmComm::GetRank();
+        int num_procs = IssmComm::GetSize();
+
+        /*retrieve node info*/
+        int glocalsize         = this->NumberOfDofsLocalAll(GsetEnum);
+        int glocalsize_masters = this->NumberOfDofsLocal(GsetEnum);
+        int maxdofspernode     = this->MaxNumDofs(GsetEnum);
+
+        /*Get local vector of ug*/
+        int        *indices_ug_masters = NULL;
+        IssmDouble *local_ug_masters   = NULL;
+        ug->GetLocalVector(&local_ug_masters,&indices_ug_masters);
+        _assert_(glocalsize_masters==indices_ug_masters[glocalsize_masters-1] - indices_ug_masters[0]+1);
+        xDelete<int>(indices_ug_masters);
+
+        /*Now, extend vectors to account for clones (make vectors longer, for clones at the end)*/
+        IssmDouble *local_ug  = xNew<IssmDouble>(glocalsize);
+        xMemCpy<IssmDouble>(local_ug,local_ug_masters,glocalsize_masters);
+        xDelete<IssmDouble>(local_ug_masters);
+
+        /*Now send and receive ug for nodes on partition edge*/
+        #ifdef _HAVE_AD_
+        IssmDouble* buffer = xNew<IssmDouble>(this->Size()*maxdofspernode,"t"); //only one alloc, "t" is required by adolc
+        #else
+        IssmDouble* buffer = xNew<IssmDouble>(this->Size()*maxdofspernode);
+        #endif
+        for(int rank=0;rank<num_procs;rank++){
+                if(this->common_send[rank]){
+                        int  numids = this->common_send[rank];
+                        for(int i=0;i<numids;i++){
+                                int   master_lid = this->common_send_ids[rank][i];
+                                Node* node=xDynamicCast<Node*>(this->GetObjectByOffset(master_lid));
+                                _assert_(!node->IsClone());
+                                for(int j=0;j<node->gsize;j++) buffer[i*maxdofspernode+j]=local_ug[node->gdoflist_local[j]];
+                        }
+                        ISSM_MPI_Send(buffer,numids*maxdofspernode,ISSM_MPI_DOUBLE,rank,0,IssmComm::GetComm());
+                }
+        }
+        for(int rank=0;rank<num_procs;rank++){
+                if(this->common_recv[rank]){
+                        int  numids = this->common_recv[rank];
+                        ISSM_MPI_Recv(buffer,numids*maxdofspernode,ISSM_MPI_DOUBLE,rank,0,IssmComm::GetComm(),&status);
+                        for(int i=0;i<numids;i++){
+                                int   master_lid = this->common_recv_ids[rank][i];
+                                Node* node=xDynamicCast<Node*>(this->GetObjectByOffset(master_lid));
+                                for(int j=0;j<node->gsize;j++) local_ug[node->gdoflist_local[j]] = buffer[i*maxdofspernode+j];
+                        }
+                }
+        }
+        xDelete<IssmDouble>(buffer);
+
+        /*Assign output pointer*/
+        *plocal_ug = local_ug;
+}/*}}}*/

issm/trunk-jpl/src/c/classes/Nodes.h

@@ -3 +3 @@
 
 #include "../datastructures/datastructures.h"
+#include "../toolkits/toolkits.h"
 class Parameters;
 class Elements;
@@ -47 +48 @@
                 int   NumberOfNodesLocalAll(void);
                 bool  RequiresDofReindexing(void);
+                void  CheckDofListAcrossPartitions(void);
+                void  GetLocalVectorWithClonesGset(IssmDouble** plocal_vector,Vector<IssmDouble> *vector);
 };