Index: /issm/trunk-jpl/src/c/classes/Elements/Penta.cpp =================================================================== --- /issm/trunk-jpl/src/c/classes/Elements/Penta.cpp (revision 23637) +++ /issm/trunk-jpl/src/c/classes/Elements/Penta.cpp (revision 23638) @@ -1662,4 +1662,12 @@ switch(type){ + case VertexLIdEnum: + for (int i=0;ivertices[i]->Lid()]; + } + /*update input*/ + this->inputs->AddInput(new PentaInput(name,values,P1Enum)); + return; + case VertexPIdEnum: for (int i=0;i *vector){/*{{{*/ + + /*recover my_rank:*/ + ISSM_MPI_Status status; + int my_rank = IssmComm::GetRank(); + int num_procs = IssmComm::GetSize(); + + /*retrieve vertex info*/ + int localsize = this->vertices->NumberOfVerticesLocalAll(); + int localsize_masters = this->vertices->NumberOfVerticesLocal(); + + /*Get local vector of vector*/ + int *indices_vector_masters = NULL; + IssmDouble *local_vector_masters = NULL; + vector->GetLocalVector(&local_vector_masters,&indices_vector_masters); + _assert_(localsize_masters==indices_vector_masters[localsize_masters-1] - indices_vector_masters[0]+1); + xDelete(indices_vector_masters); + + /*Now, extend vectors to account for clones (make vectors longer, for clones at the end)*/ + IssmDouble *local_vector = xNew(localsize); + xMemCpy(local_vector,local_vector_masters,localsize_masters); + xDelete(local_vector_masters); + + /*Now send and receive vector for vertices on partition edge*/ + #ifdef _HAVE_AD_ + IssmDouble* buffer = xNew(this->vertices->Size(),"t"); //only one alloc, "t" is required by adolc + #else + IssmDouble* buffer = xNew(this->vertices->Size()); + #endif + for(int rank=0;rankvertices->common_send[rank]){ + int numids = this->vertices->common_send[rank]; + for(int i=0;ivertices->common_send_ids[rank][i]; + Vertex* vertex=xDynamicCast(this->vertices->GetObjectByOffset(master_lid)); + _assert_(!vertex->clone); + buffer[i] = local_vector[vertex->lid]; + } + ISSM_MPI_Send(buffer,numids,ISSM_MPI_DOUBLE,rank,0,IssmComm::GetComm()); + } + } + for(int rank=0;rankvertices->common_recv[rank]){ + int numids = this->vertices->common_recv[rank]; + ISSM_MPI_Recv(buffer,numids,ISSM_MPI_DOUBLE,rank,0,IssmComm::GetComm(),&status); + for(int i=0;ivertices->common_recv_ids[rank][i]; + Vertex* vertex=xDynamicCast(this->vertices->GetObjectByOffset(master_lid)); + _assert_(vertex->clone); + local_vector[vertex->lid] = buffer[i]; + } + } + } + xDelete(buffer); + + /*Assign output pointer*/ + *plocal_vector = local_vector; +}/*}}}*/ void FemModel::GroundedAreax(IssmDouble* pV, bool scaled){/*{{{*/ @@ -2616,7 +2674,9 @@ /*Allocate vector*/ - Vector *vx=new Vector(vertices->NumberOfVertices()); - Vector *vy=new Vector(vertices->NumberOfVertices()); - Vector *vz=new Vector(vertices->NumberOfVertices()); + int numvert = vertices->NumberOfVertices(); + int numvert_local = vertices->NumberOfVerticesLocal(); + Vector *vx=new Vector(numvert_local,numvert); + Vector *vy=new Vector(numvert_local,numvert); + Vector *vz=new Vector(numvert_local,numvert); /*Update verices new geometry: */ Index: /issm/trunk-jpl/src/c/classes/FemModel.h =================================================================== --- /issm/trunk-jpl/src/c/classes/FemModel.h (revision 23637) +++ /issm/trunk-jpl/src/c/classes/FemModel.h (revision 23638) @@ -97,4 +97,5 @@ void GetInputLocalMinMaxOnNodesx(IssmDouble** pmin,IssmDouble** pmax,IssmDouble* ug); void GetLocalVectorWithClonesGset(IssmDouble** plocal_ug,Vector *ug); + void GetLocalVectorWithClonesVertices(IssmDouble** plocal_vector,Vector *vector); void GroundedAreax(IssmDouble* pV, bool scaled); void IceMassx(IssmDouble* pV, bool scaled); Index: /issm/trunk-jpl/src/c/classes/Nodes.cpp =================================================================== --- /issm/trunk-jpl/src/c/classes/Nodes.cpp (revision 23637) +++ /issm/trunk-jpl/src/c/classes/Nodes.cpp (revision 23638) @@ -171,5 +171,4 @@ for(int i=0;iSize();i++){ - /*Check that this node corresponds to our analysis currently being carried out: */ Node* node=xDynamicCast(this->GetObjectByOffset(i)); node->DistributeGlobalDofsMasters(offset,setenum); Index: /issm/trunk-jpl/src/c/classes/Vertex.cpp =================================================================== --- /issm/trunk-jpl/src/c/classes/Vertex.cpp (revision 23637) +++ /issm/trunk-jpl/src/c/classes/Vertex.cpp (revision 23638) @@ -20,5 +20,5 @@ } /*}}}*/ -Vertex::Vertex(int vertex_id, int vertex_sid,int vertex_lid,int vertex_pid,bool vertex_clone, IoModel* iomodel,bool isamr){/*{{{*/ +Vertex::Vertex(int vertex_id, int vertex_sid,int vertex_pid,bool vertex_clone, IoModel* iomodel,bool isamr){/*{{{*/ /*Checks in debugging mode*/ @@ -29,5 +29,5 @@ this->sid = vertex_sid; this->pid = vertex_pid; - this->lid = vertex_lid; + this->lid = -1; /*Assigned later*/ this->clone = vertex_clone; Index: /issm/trunk-jpl/src/c/classes/Vertex.h =================================================================== --- /issm/trunk-jpl/src/c/classes/Vertex.h (revision 23637) +++ /issm/trunk-jpl/src/c/classes/Vertex.h (revision 23638) @@ -37,5 +37,5 @@ /*Vertex constructors, destructors {{{*/ Vertex(); - Vertex(int id, int sid,int lid,int pid,bool clone, IoModel* iomodel,bool isamr); + Vertex(int id, int sid,int pid,bool clone, IoModel* iomodel,bool isamr); ~Vertex(); /*}}}*/ Index: /issm/trunk-jpl/src/c/classes/Vertices.cpp =================================================================== --- /issm/trunk-jpl/src/c/classes/Vertices.cpp (revision 23637) +++ /issm/trunk-jpl/src/c/classes/Vertices.cpp (revision 23638) @@ -25,9 +25,12 @@ /*Object constructors and destructor*/ Vertices::Vertices(){/*{{{*/ - this->enum_type = VerticesEnum; - this->common_recv = NULL; - this->common_recv_ids = NULL; - this->common_send = NULL; - this->common_send_ids = NULL; + this->enum_type = VerticesEnum; + this->common_recv = NULL; + this->common_recv_ids = NULL; + this->common_send = NULL; + this->common_send_ids = NULL; + this->numberofvertices = -1; + this->numberofvertices_local = -1; + this->numberofmasters_local = -1; return; } @@ -127,29 +130,4 @@ } /*}}}*/ -int Vertices::NumberOfVertices(void){/*{{{*/ - - int i,sid; - int max_sid=0; - int vertex_max_sid; - - if(this->Size()==0) return 0; - - for(i=0;iSize();i++){ - Vertex* vertex=xDynamicCast(this->GetObjectByOffset(i)); - sid=vertex->Sid(); - if (sid>max_sid)max_sid=sid; - } - - ISSM_MPI_Reduce (&max_sid,&vertex_max_sid,1,ISSM_MPI_INT,ISSM_MPI_MAX,0,IssmComm::GetComm() ); - ISSM_MPI_Bcast(&vertex_max_sid,1,ISSM_MPI_INT,0,IssmComm::GetComm()); - max_sid=vertex_max_sid; - - /*sid starts at 0*/ - max_sid++; - - /*return:*/ - return max_sid; -} -/*}}}*/ void Vertices::LatLonList(IssmDouble** plat,IssmDouble** plon){/*{{{*/ @@ -189,2 +167,82 @@ } /*}}}*/ + +void Vertices::Finalize(){/*{{{*/ + + /*recover my_rank:*/ + ISSM_MPI_Status status; + int my_rank = IssmComm::GetRank(); + int num_procs = IssmComm::GetSize(); + + /*1. set counters*/ + this->numberofvertices_local=this->Size(); + this->numberofmasters_local=0; + for(int i=0;iSize();i++){ + Vertex* vertex=xDynamicCast(this->GetObjectByOffset(i)); + if(!vertex->clone) this->numberofmasters_local++; + } + ISSM_MPI_Allreduce((void*)&this->numberofmasters_local,(void*)&this->numberofvertices,1,ISSM_MPI_INT,ISSM_MPI_SUM,IssmComm::GetComm()); + + /*2. Distribute lids (First: masters, then clones)*/ + int lid = 0; + for(int i=0;iSize();i++){ + Vertex* vertex=xDynamicCast(this->GetObjectByOffset(i)); + if(!vertex->clone) vertex->lid=lid++; + } + for(int i=0;iSize();i++){ + Vertex* vertex=xDynamicCast(this->GetObjectByOffset(i)); + if(vertex->clone) vertex->lid=lid++; + } + + /* Now every object has distributed dofs, but locally, and with a dof count starting from + * 0. This means the dofs between all the cpus are not unique. We now offset the dofs of each + * cpus by the total last (master) dofs of the previus cpu, starting from 0. + * First: get number of dofs for each cpu*/ + int* all_num_masters=xNew(num_procs); + ISSM_MPI_Gather(&this->numberofmasters_local,1,ISSM_MPI_INT,all_num_masters,1,ISSM_MPI_INT,0,IssmComm::GetComm()); + ISSM_MPI_Bcast(all_num_masters,num_procs,ISSM_MPI_INT,0,IssmComm::GetComm()); + + /* Every cpu should start its own dof count at the end of the dofcount from cpu-1*/ + int offset=0; + for(int i=0;i(all_num_masters); + for(int i=0;iSize();i++){ + Vertex* vertex=xDynamicCast(this->GetObjectByOffset(i)); + vertex->pid = vertex->lid+offset; + } + + /* Finally, remember that cpus may have skipped some objects, because they were clones. For every + * object that is not a clone, tell them to show their dofs, so that later on, they can get picked + * up by their clones: */ + int* truepids = xNew(this->Size()); //only one alloc + for(int rank=0;rankcommon_send[rank]){ + int numids = this->common_send[rank]; + for(int i=0;i(this->GetObjectByOffset(this->common_send_ids[rank][i])); + truepids[i] = vertex->pid; + } + ISSM_MPI_Send(truepids,numids,ISSM_MPI_INT,rank,0,IssmComm::GetComm()); + } + } + for(int rank=0;rankcommon_recv[rank]){ + int numids = this->common_recv[rank]; + ISSM_MPI_Recv(truepids,numids,ISSM_MPI_INT,rank,0,IssmComm::GetComm(),&status); + for(int i=0;i(this->GetObjectByOffset(this->common_recv_ids[rank][i])); + vertex->pid = truepids[i]; + } + } + } + xDelete(truepids); +}/*}}}*/ +int Vertices::NumberOfVertices(){/*{{{*/ + return this->numberofvertices; +}/*}}}*/ +int Vertices::NumberOfVerticesLocal(void){/*{{{*/ + return this->numberofmasters_local; +}/*}}}*/ +int Vertices::NumberOfVerticesLocalAll(void){/*{{{*/ + return this->numberofvertices_local; +}/*}}}*/ Index: /issm/trunk-jpl/src/c/classes/Vertices.h =================================================================== --- /issm/trunk-jpl/src/c/classes/Vertices.h (revision 23637) +++ /issm/trunk-jpl/src/c/classes/Vertices.h (revision 23638) @@ -14,4 +14,8 @@ class Vertices: public DataSet{ + private: + int numberofvertices; + int numberofvertices_local; + int numberofmasters_local; public: int* common_recv; @@ -26,8 +30,11 @@ /*Objects virtual functions*/ Vertices* Copy(); - void Marshall(char** pmarshalled_data,int* pmarshalled_data_size, int marshall_direction); + void Marshall(char** pmarshalled_data,int* pmarshalled_data_size, int marshall_direction); /*numerics:*/ + void Finalize(void); int NumberOfVertices(void); + int NumberOfVerticesLocal(void); + int NumberOfVerticesLocalAll(void); void LatLonList(IssmDouble** lat,IssmDouble** lon); }; Index: /issm/trunk-jpl/src/c/modules/InputUpdateFromVectorx/InputUpdateFromVectorx.cpp =================================================================== --- /issm/trunk-jpl/src/c/modules/InputUpdateFromVectorx/InputUpdateFromVectorx.cpp (revision 23637) +++ /issm/trunk-jpl/src/c/modules/InputUpdateFromVectorx/InputUpdateFromVectorx.cpp (revision 23638) @@ -9,7 +9,15 @@ void InputUpdateFromVectorx(FemModel* femmodel,Vector* vector, int name, int type){ - IssmDouble* serial_vector=vector->ToMPISerial(); - InputUpdateFromVectorx(femmodel,serial_vector,name,type); - xDelete(serial_vector); + if(type==VertexPIdEnum){ + IssmDouble* serial_vector=NULL; + femmodel->GetLocalVectorWithClonesVertices(&serial_vector,vector); + InputUpdateFromVectorx(femmodel,serial_vector,name,VertexLIdEnum); + xDelete(serial_vector); + } + else{ + IssmDouble* serial_vector=vector->ToMPISerial(); + InputUpdateFromVectorx(femmodel,serial_vector,name,type); + xDelete(serial_vector); + } } Index: /issm/trunk-jpl/src/c/modules/ModelProcessorx/CreateElementsVerticesAndMaterials.cpp =================================================================== --- /issm/trunk-jpl/src/c/modules/ModelProcessorx/CreateElementsVerticesAndMaterials.cpp (revision 23637) +++ /issm/trunk-jpl/src/c/modules/ModelProcessorx/CreateElementsVerticesAndMaterials.cpp (revision 23638) @@ -411,5 +411,5 @@ if(vertices_lids[i]!=-1){ bool isclone = (vertices_ranks[MAXCONNECTIVITY*i+0]!=my_rank); - vertices->AddObject(new Vertex(i+1,i,vertices_lids[i],vertices_pids[i],isclone,iomodel,isamr)); + vertices->AddObject(new Vertex(i+1,i,vertices_pids[i],isclone,iomodel,isamr)); } } @@ -424,5 +424,5 @@ if(vertices_lids[i]!=-1){ bool isclone = (vertices_ranks[MAXCONNECTIVITY*i+0]!=my_rank); - vertices->AddObject(new Vertex(i+1,i,vertices_lids[i],vertices_pids[i],isclone,iomodel,isamr)); + vertices->AddObject(new Vertex(i+1,i,vertices_pids[i],isclone,iomodel,isamr)); } } @@ -453,3 +453,6 @@ vertices->common_send_ids=common_send_ids; vertices->common_recv_ids=common_recv_ids; -}/*}}}*/ + + /*Finalize Initialization*/ + vertices->Finalize(); +}/*}}}*/