Changeset 23638

Timestamp:

01/16/19 16:54:18 (6 years ago)

Author:

Mathieu Morlighem

Message:

NEW: improved scalability of InputUpdateFromVector if on PIDs

Location:

issm/trunk-jpl/src/c

Files:

• classes/Elements/Penta.cpp (modified) (1 diff)
• classes/FemModel.cpp (modified) (2 diffs)
• classes/FemModel.h (modified) (1 diff)
• classes/Nodes.cpp (modified) (1 diff)
• classes/Vertex.cpp (modified) (2 diffs)
• classes/Vertex.h (modified) (1 diff)
• classes/Vertices.cpp (modified) (3 diffs)
• classes/Vertices.h (modified) (2 diffs)
• modules/InputUpdateFromVectorx/InputUpdateFromVectorx.cpp (modified) (1 diff)
• modules/ModelProcessorx/CreateElementsVerticesAndMaterials.cpp (modified) (3 diffs)

issm/trunk-jpl/src/c/classes/Elements/Penta.cpp

@@ -1662 +1662 @@
 
         switch(type){
+                case VertexLIdEnum: 
+                        for (int i=0;i<NUMVERTICES;i++){
+                                values[i]=vector[this->vertices[i]->Lid()];
+                        }
+                        /*update input*/
+                        this->inputs->AddInput(new PentaInput(name,values,P1Enum));
+                        return;
+
                 case VertexPIdEnum: 
                         for (int i=0;i<NUMVERTICES;i++){

issm/trunk-jpl/src/c/classes/FemModel.cpp

@@ -1364 +1364 @@
         *plocal_ug = local_ug;
 }/*}}}*/
+void FemModel::GetLocalVectorWithClonesVertices(IssmDouble** plocal_vector,Vector<IssmDouble> *vector){/*{{{*/
+
+        /*recover my_rank:*/
+        ISSM_MPI_Status status;
+        int my_rank   = IssmComm::GetRank();
+        int num_procs = IssmComm::GetSize();
+
+        /*retrieve vertex info*/
+        int localsize         = this->vertices->NumberOfVerticesLocalAll();
+        int localsize_masters = this->vertices->NumberOfVerticesLocal();
+
+        /*Get local vector of vector*/
+        int        *indices_vector_masters = NULL;
+        IssmDouble *local_vector_masters   = NULL;
+        vector->GetLocalVector(&local_vector_masters,&indices_vector_masters);
+        _assert_(localsize_masters==indices_vector_masters[localsize_masters-1] - indices_vector_masters[0]+1);
+        xDelete<int>(indices_vector_masters);
+
+        /*Now, extend vectors to account for clones (make vectors longer, for clones at the end)*/
+        IssmDouble *local_vector  = xNew<IssmDouble>(localsize);
+        xMemCpy<IssmDouble>(local_vector,local_vector_masters,localsize_masters);
+        xDelete<IssmDouble>(local_vector_masters);
+
+        /*Now send and receive vector for vertices on partition edge*/
+        #ifdef _HAVE_AD_
+        IssmDouble* buffer = xNew<IssmDouble>(this->vertices->Size(),"t"); //only one alloc, "t" is required by adolc
+        #else
+        IssmDouble* buffer = xNew<IssmDouble>(this->vertices->Size());
+        #endif
+        for(int rank=0;rank<num_procs;rank++){
+                if(this->vertices->common_send[rank]){
+                        int  numids = this->vertices->common_send[rank];
+                        for(int i=0;i<numids;i++){
+                                int   master_lid = this->vertices->common_send_ids[rank][i];
+                                Vertex* vertex=xDynamicCast<Vertex*>(this->vertices->GetObjectByOffset(master_lid));
+                                _assert_(!vertex->clone);
+                                buffer[i] = local_vector[vertex->lid];
+                        }
+                        ISSM_MPI_Send(buffer,numids,ISSM_MPI_DOUBLE,rank,0,IssmComm::GetComm());
+                }
+        }
+        for(int rank=0;rank<num_procs;rank++){
+                if(this->vertices->common_recv[rank]){
+                        int  numids = this->vertices->common_recv[rank];
+                        ISSM_MPI_Recv(buffer,numids,ISSM_MPI_DOUBLE,rank,0,IssmComm::GetComm(),&status);
+                        for(int i=0;i<numids;i++){
+                                int   master_lid = this->vertices->common_recv_ids[rank][i];
+                                Vertex* vertex=xDynamicCast<Vertex*>(this->vertices->GetObjectByOffset(master_lid));
+                                _assert_(vertex->clone);
+                                local_vector[vertex->lid] = buffer[i];
+                        }
+                }
+        }
+        xDelete<IssmDouble>(buffer);
+
+        /*Assign output pointer*/
+        *plocal_vector = local_vector;
+}/*}}}*/
 void FemModel::GroundedAreax(IssmDouble* pV, bool scaled){/*{{{*/
 
@@ -2616 +2674 @@
 
         /*Allocate vector*/
-        Vector<IssmDouble> *vx=new Vector<IssmDouble>(vertices->NumberOfVertices());
-        Vector<IssmDouble> *vy=new Vector<IssmDouble>(vertices->NumberOfVertices());
-        Vector<IssmDouble> *vz=new Vector<IssmDouble>(vertices->NumberOfVertices());
+        int numvert       = vertices->NumberOfVertices();
+        int numvert_local = vertices->NumberOfVerticesLocal();
+        Vector<IssmDouble> *vx=new Vector<IssmDouble>(numvert_local,numvert);
+        Vector<IssmDouble> *vy=new Vector<IssmDouble>(numvert_local,numvert);
+        Vector<IssmDouble> *vz=new Vector<IssmDouble>(numvert_local,numvert);
 
         /*Update verices new geometry: */

issm/trunk-jpl/src/c/classes/FemModel.h

@@ -97 +97 @@
                 void GetInputLocalMinMaxOnNodesx(IssmDouble** pmin,IssmDouble** pmax,IssmDouble* ug);
                 void GetLocalVectorWithClonesGset(IssmDouble** plocal_ug,Vector<IssmDouble> *ug);
+                void GetLocalVectorWithClonesVertices(IssmDouble** plocal_vector,Vector<IssmDouble> *vector);
                 void GroundedAreax(IssmDouble* pV, bool scaled);
                 void IceMassx(IssmDouble* pV, bool scaled);

issm/trunk-jpl/src/c/classes/Nodes.cpp

@@ -171 +171 @@
 
         for(int i=0;i<this->Size();i++){
-                /*Check that this node corresponds to our analysis currently being carried out: */
                 Node* node=xDynamicCast<Node*>(this->GetObjectByOffset(i));
                 node->DistributeGlobalDofsMasters(offset,setenum);

issm/trunk-jpl/src/c/classes/Vertex.cpp

@@ -20 +20 @@
 }
 /*}}}*/
-Vertex::Vertex(int vertex_id, int vertex_sid,int vertex_lid,int vertex_pid,bool vertex_clone, IoModel* iomodel,bool isamr){/*{{{*/
+Vertex::Vertex(int vertex_id, int vertex_sid,int vertex_pid,bool vertex_clone, IoModel* iomodel,bool isamr){/*{{{*/
 
         /*Checks in debugging mode*/
@@ -29 +29 @@
         this->sid   = vertex_sid;
         this->pid   = vertex_pid;
-        this->lid   = vertex_lid;
+        this->lid   = -1; /*Assigned later*/
         this->clone = vertex_clone;
 

issm/trunk-jpl/src/c/classes/Vertex.h

@@ -37 +37 @@
                 /*Vertex constructors, destructors {{{*/
                 Vertex();
-                Vertex(int id, int sid,int lid,int pid,bool clone, IoModel* iomodel,bool isamr);
+                Vertex(int id, int sid,int pid,bool clone, IoModel* iomodel,bool isamr);
                 ~Vertex();
                 /*}}}*/

issm/trunk-jpl/src/c/classes/Vertices.cpp

@@ -25 +25 @@
 /*Object constructors and destructor*/
 Vertices::Vertices(){/*{{{*/
-        this->enum_type       = VerticesEnum;
-        this->common_recv     = NULL;
-        this->common_recv_ids = NULL;
-        this->common_send     = NULL;
-        this->common_send_ids = NULL;
+        this->enum_type              = VerticesEnum;
+        this->common_recv            = NULL;
+        this->common_recv_ids        = NULL;
+        this->common_send            = NULL;
+        this->common_send_ids        = NULL;
+        this->numberofvertices       = -1;
+        this->numberofvertices_local = -1;
+        this->numberofmasters_local  = -1;
         return;
 }
@@ -127 +130 @@
 }
 /*}}}*/
-int Vertices::NumberOfVertices(void){/*{{{*/
-
-        int i,sid;
-        int max_sid=0;
-        int vertex_max_sid;
-
-        if(this->Size()==0) return 0;
-
-        for(i=0;i<this->Size();i++){
-                Vertex* vertex=xDynamicCast<Vertex*>(this->GetObjectByOffset(i));
-                sid=vertex->Sid();
-                if (sid>max_sid)max_sid=sid;
-        }
-
-        ISSM_MPI_Reduce (&max_sid,&vertex_max_sid,1,ISSM_MPI_INT,ISSM_MPI_MAX,0,IssmComm::GetComm() );
-        ISSM_MPI_Bcast(&vertex_max_sid,1,ISSM_MPI_INT,0,IssmComm::GetComm());
-        max_sid=vertex_max_sid;
-
-        /*sid starts at 0*/
-        max_sid++;
-
-        /*return:*/
-        return max_sid;
-}
-/*}}}*/
 void Vertices::LatLonList(IssmDouble** plat,IssmDouble** plon){/*{{{*/
 
@@ -189 +167 @@
 }
 /*}}}*/
+
+void Vertices::Finalize(){/*{{{*/
+
+        /*recover my_rank:*/
+        ISSM_MPI_Status status;
+        int my_rank   = IssmComm::GetRank();
+        int num_procs = IssmComm::GetSize();
+
+        /*1. set counters*/
+        this->numberofvertices_local=this->Size();
+        this->numberofmasters_local=0;
+        for(int i=0;i<this->Size();i++){
+                Vertex* vertex=xDynamicCast<Vertex*>(this->GetObjectByOffset(i));
+                if(!vertex->clone) this->numberofmasters_local++;
+        }
+        ISSM_MPI_Allreduce((void*)&this->numberofmasters_local,(void*)&this->numberofvertices,1,ISSM_MPI_INT,ISSM_MPI_SUM,IssmComm::GetComm());
+
+        /*2. Distribute lids (First: masters, then clones)*/
+        int lid = 0;
+        for(int i=0;i<this->Size();i++){
+                Vertex* vertex=xDynamicCast<Vertex*>(this->GetObjectByOffset(i));
+                if(!vertex->clone) vertex->lid=lid++;
+        }
+        for(int i=0;i<this->Size();i++){
+                Vertex* vertex=xDynamicCast<Vertex*>(this->GetObjectByOffset(i));
+                if(vertex->clone) vertex->lid=lid++;
+        }
+
+        /* Now every object has distributed dofs, but locally, and with a dof count starting from 
+         * 0. This means the dofs between all the cpus are not unique. We now offset the dofs of each
+         * cpus by the total last (master) dofs of the previus cpu, starting from 0.
+         * First: get number of dofs for each cpu*/
+        int* all_num_masters=xNew<int>(num_procs);
+        ISSM_MPI_Gather(&this->numberofmasters_local,1,ISSM_MPI_INT,all_num_masters,1,ISSM_MPI_INT,0,IssmComm::GetComm());
+        ISSM_MPI_Bcast(all_num_masters,num_procs,ISSM_MPI_INT,0,IssmComm::GetComm());
+
+        /* Every cpu should start its own dof count at the end of the dofcount from cpu-1*/
+        int offset=0;
+        for(int i=0;i<my_rank;i++) offset+=all_num_masters[i];
+        xDelete<int>(all_num_masters);
+        for(int i=0;i<this->Size();i++){
+                Vertex* vertex=xDynamicCast<Vertex*>(this->GetObjectByOffset(i));
+                vertex->pid = vertex->lid+offset;
+        }
+
+        /* Finally, remember that cpus may have skipped some objects, because they were clones. For every 
+         * object that is not a clone, tell them to show their dofs, so that later on, they can get picked 
+         * up by their clones: */
+        int* truepids = xNew<int>(this->Size()); //only one alloc
+        for(int rank=0;rank<num_procs;rank++){
+                if(this->common_send[rank]){
+                        int  numids = this->common_send[rank];
+                        for(int i=0;i<numids;i++){
+                                Vertex* vertex=xDynamicCast<Vertex*>(this->GetObjectByOffset(this->common_send_ids[rank][i]));
+                                truepids[i] = vertex->pid;
+                        }
+                        ISSM_MPI_Send(truepids,numids,ISSM_MPI_INT,rank,0,IssmComm::GetComm());
+                }
+        }
+        for(int rank=0;rank<num_procs;rank++){
+                if(this->common_recv[rank]){
+                        int  numids = this->common_recv[rank];
+                        ISSM_MPI_Recv(truepids,numids,ISSM_MPI_INT,rank,0,IssmComm::GetComm(),&status);
+                        for(int i=0;i<numids;i++){
+                                Vertex* vertex=xDynamicCast<Vertex*>(this->GetObjectByOffset(this->common_recv_ids[rank][i]));
+                                vertex->pid = truepids[i];
+                        }
+                }
+        }
+        xDelete<int>(truepids);
+}/*}}}*/
+int Vertices::NumberOfVertices(){/*{{{*/
+        return this->numberofvertices;
+}/*}}}*/
+int Vertices::NumberOfVerticesLocal(void){/*{{{*/
+        return this->numberofmasters_local;
+}/*}}}*/
+int Vertices::NumberOfVerticesLocalAll(void){/*{{{*/
+        return this->numberofvertices_local;
+}/*}}}*/

issm/trunk-jpl/src/c/classes/Vertices.h

@@ -14 +14 @@
 class Vertices: public DataSet{
 
+        private:
+                int numberofvertices;
+                int numberofvertices_local;
+                int numberofmasters_local;
         public:
                 int*  common_recv;
@@ -26 +30 @@
                 /*Objects virtual functions*/
                 Vertices* Copy();
-                void Marshall(char** pmarshalled_data,int* pmarshalled_data_size, int marshall_direction);
+                void      Marshall(char** pmarshalled_data,int* pmarshalled_data_size, int marshall_direction);
 
                 /*numerics:*/
+                void  Finalize(void);
                 int   NumberOfVertices(void);
+                int   NumberOfVerticesLocal(void);
+                int   NumberOfVerticesLocalAll(void);
                 void  LatLonList(IssmDouble** lat,IssmDouble** lon);
 };

issm/trunk-jpl/src/c/modules/InputUpdateFromVectorx/InputUpdateFromVectorx.cpp

@@ -9 +9 @@
 void InputUpdateFromVectorx(FemModel* femmodel,Vector<IssmDouble>* vector, int name, int type){
 
-        IssmDouble* serial_vector=vector->ToMPISerial();
-        InputUpdateFromVectorx(femmodel,serial_vector,name,type);
-        xDelete<IssmDouble>(serial_vector);
+        if(type==VertexPIdEnum){
+                IssmDouble* serial_vector=NULL;
+                femmodel->GetLocalVectorWithClonesVertices(&serial_vector,vector);
+                InputUpdateFromVectorx(femmodel,serial_vector,name,VertexLIdEnum);
+                xDelete<IssmDouble>(serial_vector);
+        }
+        else{
+                IssmDouble* serial_vector=vector->ToMPISerial();
+                InputUpdateFromVectorx(femmodel,serial_vector,name,type);
+                xDelete<IssmDouble>(serial_vector);
+        }
 }
 

issm/trunk-jpl/src/c/modules/ModelProcessorx/CreateElementsVerticesAndMaterials.cpp

@@ -411 +411 @@
                         if(vertices_lids[i]!=-1){
                                 bool isclone = (vertices_ranks[MAXCONNECTIVITY*i+0]!=my_rank);
-                                vertices->AddObject(new Vertex(i+1,i,vertices_lids[i],vertices_pids[i],isclone,iomodel,isamr));
+                                vertices->AddObject(new Vertex(i+1,i,vertices_pids[i],isclone,iomodel,isamr));
                         }
                 }
@@ -424 +424 @@
                         if(vertices_lids[i]!=-1){
                                 bool isclone = (vertices_ranks[MAXCONNECTIVITY*i+0]!=my_rank);
-                                vertices->AddObject(new Vertex(i+1,i,vertices_lids[i],vertices_pids[i],isclone,iomodel,isamr));
+                                vertices->AddObject(new Vertex(i+1,i,vertices_pids[i],isclone,iomodel,isamr));
                         }
                 }
@@ -453 +453 @@
         vertices->common_send_ids=common_send_ids;
         vertices->common_recv_ids=common_recv_ids;
-}/*}}}*/
+
+        /*Finalize Initialization*/
+        vertices->Finalize();
+}/*}}}*/