Index: /issm/trunk-jpl/src/c/analyses/AdjointHorizAnalysis.cpp =================================================================== --- /issm/trunk-jpl/src/c/analyses/AdjointHorizAnalysis.cpp (revision 23628) +++ /issm/trunk-jpl/src/c/analyses/AdjointHorizAnalysis.cpp (revision 23629) @@ -2335,6 +2335,6 @@ /*Get dof list: */ - element->GetDofListVelocity(&vdoflist,GsetEnum); - element->GetDofListPressure(&pdoflist,GsetEnum); + element->GetDofListLocalVelocity(&vdoflist,GsetEnum); + element->GetDofListLocalPressure(&pdoflist,GsetEnum); /*Use the dof list to index into the solution vector: */ @@ -2401,5 +2401,5 @@ else numdof = numnodes*1; /*Fetch dof list and allocate solution vectors*/ - element->GetDofList(&doflist,NoneApproximationEnum,GsetEnum); + element->GetDofListLocal(&doflist,NoneApproximationEnum,GsetEnum); IssmDouble* values = xNew(numdof); IssmDouble* lambdax = xNew(numnodes); Index: /issm/trunk-jpl/src/c/analyses/DamageEvolutionAnalysis.cpp =================================================================== --- /issm/trunk-jpl/src/c/analyses/DamageEvolutionAnalysis.cpp (revision 23628) +++ /issm/trunk-jpl/src/c/analyses/DamageEvolutionAnalysis.cpp (revision 23629) @@ -634,5 +634,5 @@ /*Fetch dof list and allocate solution vector*/ - element->GetDofList(&doflist,NoneApproximationEnum,GsetEnum); + element->GetDofListLocal(&doflist,NoneApproximationEnum,GsetEnum); IssmDouble* newdamage = xNew(numnodes); Index: /issm/trunk-jpl/src/c/analyses/EnthalpyAnalysis.cpp =================================================================== --- /issm/trunk-jpl/src/c/analyses/EnthalpyAnalysis.cpp (revision 23628) +++ /issm/trunk-jpl/src/c/analyses/EnthalpyAnalysis.cpp (revision 23629) @@ -1528,5 +1528,5 @@ /*Fetch dof list and allocate solution vector*/ - element->GetDofList(&doflist,NoneApproximationEnum,GsetEnum); + element->GetDofListLocal(&doflist,NoneApproximationEnum,GsetEnum); IssmDouble* values = xNew(numnodes); IssmDouble* pressure = xNew(numnodes); Index: /issm/trunk-jpl/src/c/analyses/HydrologyDCEfficientAnalysis.cpp =================================================================== --- /issm/trunk-jpl/src/c/analyses/HydrologyDCEfficientAnalysis.cpp (revision 23628) +++ /issm/trunk-jpl/src/c/analyses/HydrologyDCEfficientAnalysis.cpp (revision 23629) @@ -435,5 +435,5 @@ IssmDouble* eplHeads = xNew(numnodes); IssmDouble* basevalue = xNew(numnodes); - basalelement->GetDofList(&doflist,NoneApproximationEnum,GsetEnum); + basalelement->GetDofListLocal(&doflist,NoneApproximationEnum,GsetEnum); Input* active_element_input=basalelement->GetInput(HydrologydcMaskEplactiveEltEnum); _assert_(active_element_input); Index: /issm/trunk-jpl/src/c/analyses/HydrologyDCInefficientAnalysis.cpp =================================================================== --- /issm/trunk-jpl/src/c/analyses/HydrologyDCInefficientAnalysis.cpp (revision 23628) +++ /issm/trunk-jpl/src/c/analyses/HydrologyDCInefficientAnalysis.cpp (revision 23629) @@ -511,5 +511,5 @@ /*Fetch dof list and allocate solution vector*/ - basalelement->GetDofList(&doflist,NoneApproximationEnum,GsetEnum); + basalelement->GetDofListLocal(&doflist,NoneApproximationEnum,GsetEnum); IssmDouble* values = xNew(numnodes); IssmDouble* pressure = xNew(numnodes); Index: /issm/trunk-jpl/src/c/analyses/HydrologyShaktiAnalysis.cpp =================================================================== --- /issm/trunk-jpl/src/c/analyses/HydrologyShaktiAnalysis.cpp (revision 23628) +++ /issm/trunk-jpl/src/c/analyses/HydrologyShaktiAnalysis.cpp (revision 23629) @@ -361,5 +361,5 @@ /*Fetch dof list and allocate solution vector*/ - element->GetDofList(&doflist,NoneApproximationEnum,GsetEnum); + element->GetDofListLocal(&doflist,NoneApproximationEnum,GsetEnum); IssmDouble* values = xNew(numnodes); Index: /issm/trunk-jpl/src/c/analyses/HydrologyShreveAnalysis.cpp =================================================================== --- /issm/trunk-jpl/src/c/analyses/HydrologyShreveAnalysis.cpp (revision 23628) +++ /issm/trunk-jpl/src/c/analyses/HydrologyShreveAnalysis.cpp (revision 23629) @@ -356,5 +356,5 @@ /*Fetch dof list and allocate solution vector*/ - element->GetDofList(&doflist,NoneApproximationEnum,GsetEnum); + element->GetDofListLocal(&doflist,NoneApproximationEnum,GsetEnum); IssmDouble* values = xNew(numnodes); Index: /issm/trunk-jpl/src/c/analyses/MasstransportAnalysis.cpp =================================================================== --- /issm/trunk-jpl/src/c/analyses/MasstransportAnalysis.cpp (revision 23628) +++ /issm/trunk-jpl/src/c/analyses/MasstransportAnalysis.cpp (revision 23629) @@ -708,5 +708,5 @@ /*Fetch dof list and allocate solution vector*/ - basalelement->GetDofList(&doflist,NoneApproximationEnum,GsetEnum); + basalelement->GetDofListLocal(&doflist,NoneApproximationEnum,GsetEnum); IssmDouble* newthickness = xNew(numnodes); IssmDouble* cumdeltathickness = xNew(numnodes); Index: /issm/trunk-jpl/src/c/analyses/StressbalanceAnalysis.cpp =================================================================== --- /issm/trunk-jpl/src/c/analyses/StressbalanceAnalysis.cpp (revision 23628) +++ /issm/trunk-jpl/src/c/analyses/StressbalanceAnalysis.cpp (revision 23629) @@ -1915,5 +1915,5 @@ /*Fetch dof list and allocate solution vectors*/ - basalelement->GetDofList(&doflist,SSAApproximationEnum,GsetEnum); + basalelement->GetDofListLocal(&doflist,SSAApproximationEnum,GsetEnum); IssmDouble* values = xNew(numdof); IssmDouble* vx = xNew(numnodes); @@ -2244,5 +2244,5 @@ /*Fetch dof list and allocate solution vectors*/ - basalelement->GetDofList(&doflist,NoneApproximationEnum,GsetEnum); + basalelement->GetDofListLocal(&doflist,NoneApproximationEnum,GsetEnum); IssmDouble* values = xNew(numdof); IssmDouble* vx = xNew(numnodes); @@ -2899,5 +2899,5 @@ /*Fetch dof list and allocate solution vectors*/ - element->GetDofList(&doflist,HOApproximationEnum,GsetEnum); + element->GetDofListLocal(&doflist,HOApproximationEnum,GsetEnum); IssmDouble* values = xNew(numdof); IssmDouble* vx = xNew(numnodes); @@ -5182,6 +5182,6 @@ /*Get dof list: */ - element->GetDofListVelocity(&vdoflist,GsetEnum); - element->GetDofListPressure(&pdoflist,GsetEnum); + element->GetDofListLocalVelocity(&vdoflist,GsetEnum); + element->GetDofListLocalPressure(&pdoflist,GsetEnum); /*Use the dof list to index into the solution vector: */ @@ -7081,7 +7081,7 @@ /*Fetch dof list and allocate solution vectors*/ - element->GetDofList(&doflistFSv,FSvelocityEnum,GsetEnum); - element->GetDofList(&doflistHO, HOApproximationEnum, GsetEnum); - element->GetDofListPressure(&doflistFSp,GsetEnum); + element->GetDofListLocal(&doflistFSv,FSvelocityEnum,GsetEnum); + element->GetDofListLocal(&doflistHO, HOApproximationEnum, GsetEnum); + element->GetDofListLocalPressure(&doflistFSp,GsetEnum); IssmDouble* HOvalues = xNew(numdofHO); IssmDouble* FSvalues = xNew(numdofFSv+numdofFSp); @@ -7180,7 +7180,7 @@ /*Fetch dof list and allocate solution vectors*/ - element->GetDofList(&doflistFSv,FSvelocityEnum,GsetEnum); - element->GetDofListPressure(&doflistFSp,GsetEnum); - basalelement->GetDofList(&doflistSSA, SSAApproximationEnum, GsetEnum); + element->GetDofListLocal(&doflistFSv,FSvelocityEnum,GsetEnum); + element->GetDofListLocalPressure(&doflistFSp,GsetEnum); + basalelement->GetDofListLocal(&doflistSSA, SSAApproximationEnum, GsetEnum); IssmDouble* SSAvalues = xNew(numdofSSA); IssmDouble* FSvalues = xNew(numdofFSv+numdofFSp); @@ -7281,6 +7281,6 @@ /*Fetch dof list and allocate solution vectors*/ - basalelement->GetDofList(&SSAdoflist,SSAApproximationEnum,GsetEnum); - element ->GetDofList(&HOdoflist, HOApproximationEnum, GsetEnum); + basalelement->GetDofListLocal(&SSAdoflist,SSAApproximationEnum,GsetEnum); + element ->GetDofListLocal(&HOdoflist, HOApproximationEnum, GsetEnum); IssmDouble* HOvalues = xNew(numdof); IssmDouble* SSAvalues = xNew(numdof); Index: /issm/trunk-jpl/src/c/analyses/StressbalanceSIAAnalysis.cpp =================================================================== --- /issm/trunk-jpl/src/c/analyses/StressbalanceSIAAnalysis.cpp (revision 23628) +++ /issm/trunk-jpl/src/c/analyses/StressbalanceSIAAnalysis.cpp (revision 23629) @@ -564,5 +564,5 @@ /*Fetch dof list and allocate solution vectors*/ - element->GetDofList(&doflist,NoneApproximationEnum,GsetEnum); + element->GetDofListLocal(&doflist,NoneApproximationEnum,GsetEnum); IssmDouble* values = xNew(numdof); IssmDouble* vx = xNew(numdof); Index: /issm/trunk-jpl/src/c/analyses/StressbalanceVerticalAnalysis.cpp =================================================================== --- /issm/trunk-jpl/src/c/analyses/StressbalanceVerticalAnalysis.cpp (revision 23628) +++ /issm/trunk-jpl/src/c/analyses/StressbalanceVerticalAnalysis.cpp (revision 23629) @@ -526,5 +526,5 @@ /*Get dof list and vertices coordinates: */ element->GetVerticesCoordinates(&xyz_list); - element->GetDofList(&doflist,NoneApproximationEnum,GsetEnum); + element->GetDofListLocal(&doflist,NoneApproximationEnum,GsetEnum); IssmDouble* values = xNew(numdof); IssmDouble* vx = xNew(numnodes); Index: /issm/trunk-jpl/src/c/analyses/ThermalAnalysis.cpp =================================================================== --- /issm/trunk-jpl/src/c/analyses/ThermalAnalysis.cpp (revision 23628) +++ /issm/trunk-jpl/src/c/analyses/ThermalAnalysis.cpp (revision 23629) @@ -814,5 +814,5 @@ /*Fetch dof list and allocate solution vector*/ - element->GetDofList(&doflist,NoneApproximationEnum,GsetEnum); + element->GetDofListLocal(&doflist,NoneApproximationEnum,GsetEnum); IssmDouble* values = xNew(numnodes); IssmDouble* surface = xNew(numnodes); Index: /issm/trunk-jpl/src/c/analyses/UzawaPressureAnalysis.cpp =================================================================== --- /issm/trunk-jpl/src/c/analyses/UzawaPressureAnalysis.cpp (revision 23628) +++ /issm/trunk-jpl/src/c/analyses/UzawaPressureAnalysis.cpp (revision 23629) @@ -205,5 +205,5 @@ /*Fetch dof list and allocate solution vector*/ - element->GetDofList(&doflist,NoneApproximationEnum,GsetEnum); + element->GetDofListLocal(&doflist,NoneApproximationEnum,GsetEnum); IssmDouble* values = xNew(numnodes); IssmDouble* valueslambda = xNewZeroInit(numnodessigma); Index: /issm/trunk-jpl/src/c/classes/Elements/Element.cpp =================================================================== --- /issm/trunk-jpl/src/c/classes/Elements/Element.cpp (revision 23628) +++ /issm/trunk-jpl/src/c/classes/Elements/Element.cpp (revision 23629) @@ -1004,4 +1004,27 @@ } /*}}}*/ +void Element::GetDofListLocal(int** pdoflist,int approximation_enum,int setenum){/*{{{*/ + + /*Fetch number of nodes and dof for this finite element*/ + int numnodes = this->GetNumberOfNodes(); + + /*First, figure out size of doflist and create it: */ + int numberofdofs=0; + for(int i=0;iGetNumberOfDofs(approximation_enum,setenum); + + /*Allocate output*/ + int* doflist=xNew(numberofdofs); + + /*Populate: */ + int count=0; + for(int i=0;iGetDofListLocal(doflist+count,approximation_enum,setenum); + count+=nodes[i]->GetNumberOfDofs(approximation_enum,setenum); + } + + /*Assign output pointers:*/ + *pdoflist=doflist; +} +/*}}}*/ void Element::GetDofListPressure(int** pdoflist,int setenum){/*{{{*/ @@ -1044,4 +1067,51 @@ for(int i=0;iGetDofList(doflist+count,FSvelocityEnum,setenum); + count+=nodes[i]->GetNumberOfDofs(FSvelocityEnum,setenum); + } + + /*Assign output pointers:*/ + *pdoflist=doflist; +} +/*}}}*/ +void Element::GetDofListLocalPressure(int** pdoflist,int setenum){/*{{{*/ + + /*Fetch number of nodes and dof for this finite element*/ + int vnumnodes = this->NumberofNodesVelocity(); + int pnumnodes = this->NumberofNodesPressure(); + + /*First, figure out size of doflist and create it: */ + int numberofdofs=0; + for(int i=vnumnodes;iGetNumberOfDofs(FSApproximationEnum,setenum); + + /*Allocate output*/ + int* doflist=xNew(numberofdofs); + + /*Populate: */ + int count=0; + for(int i=vnumnodes;iGetDofListLocal(doflist+count,FSApproximationEnum,setenum); + count+=nodes[i]->GetNumberOfDofs(FSApproximationEnum,setenum); + } + + /*Assign output pointers:*/ + *pdoflist=doflist; +} +/*}}}*/ +void Element::GetDofListLocalVelocity(int** pdoflist,int setenum){/*{{{*/ + + /*Fetch number of nodes and dof for this finite element*/ + int numnodes = this->NumberofNodesVelocity(); + + /*First, figure out size of doflist and create it: */ + int numberofdofs=0; + for(int i=0;iGetNumberOfDofs(FSvelocityEnum,setenum); + + /*Allocate output*/ + int* doflist=xNew(numberofdofs); + + /*Populate: */ + int count=0; + for(int i=0;iGetDofListLocal(doflist+count,FSvelocityEnum,setenum); count+=nodes[i]->GetNumberOfDofs(FSvelocityEnum,setenum); } Index: /issm/trunk-jpl/src/c/classes/Elements/Element.h =================================================================== --- /issm/trunk-jpl/src/c/classes/Elements/Element.h (revision 23628) +++ /issm/trunk-jpl/src/c/classes/Elements/Element.h (revision 23629) @@ -86,4 +86,7 @@ void GetDofListPressure(int** pdoflist,int setenum); void GetDofListVelocity(int** pdoflist,int setenum); + void GetDofListLocal(int** pdoflist,int approximation_enum,int setenum); + void GetDofListLocalPressure(int** pdoflist,int setenum); + void GetDofListLocalVelocity(int** pdoflist,int setenum); Input* GetInput(int inputenum); void GetInputListOnNodes(IssmDouble* pvalue,int enumtype); Index: /issm/trunk-jpl/src/c/classes/Elements/Penta.cpp =================================================================== --- /issm/trunk-jpl/src/c/classes/Elements/Penta.cpp (revision 23628) +++ /issm/trunk-jpl/src/c/classes/Elements/Penta.cpp (revision 23629) @@ -1594,5 +1594,5 @@ /*Fetch dof list and allocate solution vector*/ - GetDofList(&doflist,NoneApproximationEnum,GsetEnum); + GetDofListLocal(&doflist,NoneApproximationEnum,GsetEnum); IssmDouble* values = xNew(numnodes); @@ -1625,5 +1625,5 @@ /*Get dof list: */ - GetDofList(&doflist,NoneApproximationEnum,GsetEnum); + GetDofListLocal(&doflist,NoneApproximationEnum,GsetEnum); /*Use the dof list to index into the solution vector and extrude it */ Index: /issm/trunk-jpl/src/c/classes/Elements/Tetra.cpp =================================================================== --- /issm/trunk-jpl/src/c/classes/Elements/Tetra.cpp (revision 23628) +++ /issm/trunk-jpl/src/c/classes/Elements/Tetra.cpp (revision 23629) @@ -490,5 +490,5 @@ /*Fetch dof list and allocate solution vector*/ - GetDofList(&doflist,NoneApproximationEnum,GsetEnum); + GetDofListLocal(&doflist,NoneApproximationEnum,GsetEnum); IssmDouble* values = xNew(numnodes); Index: /issm/trunk-jpl/src/c/classes/Elements/Tria.cpp =================================================================== --- /issm/trunk-jpl/src/c/classes/Elements/Tria.cpp (revision 23628) +++ /issm/trunk-jpl/src/c/classes/Elements/Tria.cpp (revision 23629) @@ -2170,5 +2170,5 @@ /*Fetch dof list and allocate solution vector*/ - GetDofList(&doflist,NoneApproximationEnum,GsetEnum); + GetDofListLocal(&doflist,NoneApproximationEnum,GsetEnum); IssmDouble* values = xNew(numnodes); Index: /issm/trunk-jpl/src/c/classes/Node.cpp =================================================================== --- /issm/trunk-jpl/src/c/classes/Node.cpp (revision 23628) +++ /issm/trunk-jpl/src/c/classes/Node.cpp (revision 23629) @@ -452,4 +452,67 @@ } /*}}}*/ +void Node::GetDofListLocal(int* outdoflist,int approximation_enum,int setenum){/*{{{*/ + int i; + int count=0; + int count2=0; + + _assert_(!this->indexingupdate); + + if(approximation_enum==NoneApproximationEnum){ + if(setenum==GsetEnum)for(i=0;igsize;i++) outdoflist[i]=gdoflist_local[i]; + if(setenum==FsetEnum)for(i=0;ifsize;i++) outdoflist[i]=fdoflist_local[i]; + if(setenum==SsetEnum)for(i=0;issize;i++) outdoflist[i]=sdoflist_local[i]; + } + else{ + + if(setenum==GsetEnum){ + if(doftype){ + count=0; + for(i=0;igsize;i++){ + if(doftype[i]==approximation_enum){ + outdoflist[count]=gdoflist_local[i]; + count++; + } + } + _assert_(count); //at least one dof should be the approximation requested + } + else for(i=0;igsize;i++) outdoflist[i]=gdoflist_local[i]; + } + else if(setenum==FsetEnum){ + if(doftype){ + count=0; + count2=0; + for(i=0;igsize;i++){ + if(f_set[i]){ + if(doftype[i]==approximation_enum){ + outdoflist[count]=fdoflist_local[count2]; + count++; + } + count2++; + } + } + } + else for(i=0;ifsize;i++) outdoflist[i]=fdoflist_local[i]; + } + else if(setenum==SsetEnum){ + if(doftype){ + count=0; + count2=0; + for(i=0;igsize;i++){ + if(s_set[i]){ + if(doftype[i]==approximation_enum){ + outdoflist[count]=sdoflist_local[count2]; + count++; + } + count2++; + } + } + } + else for(i=0;issize;i++) outdoflist[i]=sdoflist_local[i]; + } + else _error_("set of enum type " << EnumToStringx(setenum) << " not supported yet!"); + } +} +/*}}}*/ void Node::GetLocalDofList(int* outdoflist,int approximation_enum,int setenum){/*{{{*/ int i; Index: /issm/trunk-jpl/src/c/classes/Node.h =================================================================== --- /issm/trunk-jpl/src/c/classes/Node.h (revision 23628) +++ /issm/trunk-jpl/src/c/classes/Node.h (revision 23629) @@ -89,4 +89,5 @@ int GetDof(int dofindex,int setenum); void GetDofList(int* poutdoflist,int approximation_enum,int setenum); + void GetDofListLocal(int* poutdoflist,int approximation_enum,int setenum); void GetLocalDofList(int* poutdoflist,int approximation_enum,int setenum); int GetNumberOfDofs(int approximation_enum,int setenum); Index: /issm/trunk-jpl/src/c/classes/Nodes.cpp =================================================================== --- /issm/trunk-jpl/src/c/classes/Nodes.cpp (revision 23628) +++ /issm/trunk-jpl/src/c/classes/Nodes.cpp (revision 23629) @@ -259,4 +259,15 @@ } /*}}}*/ +int Nodes::NumberOfDofsLocalAll(int setenum){/*{{{*/ + + /*go through all nodes, and get how many dofs they own, unless they are clone nodes: */ + int numdofs=0; + for(int i=0;iSize();i++){ + Node* node=xDynamicCast(this->GetObjectByOffset(i)); + numdofs+=node->GetNumberOfDofs(NoneApproximationEnum,setenum); + } + return numdofs; +} +/*}}}*/ int Nodes::NumberOfNodes(void){/*{{{*/ Index: /issm/trunk-jpl/src/c/classes/Nodes.h =================================================================== --- /issm/trunk-jpl/src/c/classes/Nodes.h (revision 23628) +++ /issm/trunk-jpl/src/c/classes/Nodes.h (revision 23629) @@ -37,4 +37,5 @@ int NumberOfDofs(int setenum); int NumberOfDofsLocal(int setenum); + int NumberOfDofsLocalAll(int setenum); int NumberOfNodes(void); bool RequiresDofReindexing(void); Index: /issm/trunk-jpl/src/c/modules/InputUpdateFromSolutionx/InputUpdateFromSolutionx.cpp =================================================================== --- /issm/trunk-jpl/src/c/modules/InputUpdateFromSolutionx/InputUpdateFromSolutionx.cpp (revision 23628) +++ /issm/trunk-jpl/src/c/modules/InputUpdateFromSolutionx/InputUpdateFromSolutionx.cpp (revision 23629) @@ -9,30 +9,70 @@ void InputUpdateFromSolutionx(FemModel* femmodel,Vector* solution){ - /*Serialize solution, so that elements can index into it on every CPU: */ - femmodel->profiler->Start(MPISERIAL); - IssmDouble* serial_solution=solution->ToMPISerial(); - femmodel->profiler->Stop(MPISERIAL); - - /*Call overloaded form of InputUpdateFromSolutionx: */ - InputUpdateFromSolutionx(femmodel,serial_solution); - - /*cleanup and return*/ - xDelete(serial_solution); -} - -void InputUpdateFromSolutionx(FemModel* femmodel,IssmDouble* solution){ - - /*retrive parameters: */ - int analysisenum; - femmodel->parameters->FindParam(&analysisenum,AnalysisTypeEnum); - /*Display message*/ if(VerboseModule()) _printf0_(" Updating inputs from solution\n"); + /*GetAnalysis*/ + int analysisenum; + femmodel->parameters->FindParam(&analysisenum,AnalysisTypeEnum); Analysis* analysis = EnumToAnalysis(analysisenum); + + /*recover my_rank:*/ + ISSM_MPI_Status status; + int my_rank = IssmComm::GetRank(); + int num_procs = IssmComm::GetSize(); + + /*retrieve node info*/ + int gsize = femmodel->nodes->NumberOfDofs(GsetEnum); + int glocalsize_masters = femmodel->nodes->NumberOfDofsLocal(GsetEnum); + int glocalsize = femmodel->nodes->NumberOfDofsLocalAll(GsetEnum); + int maxdofspernode = femmodel->nodes->MaxNumDofs(GsetEnum); + + /*Get local vector of ug*/ + int *indices_ug_masters = NULL; + IssmDouble *local_ug_masters = NULL; + solution->GetLocalVector(&local_ug_masters,&indices_ug_masters); + _assert_(glocalsize_masters==indices_ug_masters[glocalsize_masters-1] - indices_ug_masters[0]+1); + xDelete(indices_ug_masters); + + /*Now, extend vectors to account for clones (make vectors longer, for clones at the end)*/ + IssmDouble *local_ug = xNew(glocalsize); + xMemCpy(local_ug,local_ug_masters,glocalsize_masters); + xDelete(local_ug_masters); + + /*Now send and receive ug for nodes on partition edge*/ + IssmDouble* buffer = xNew(femmodel->nodes->Size()*maxdofspernode); //only one alloc + for(int rank=0;ranknodes->common_send[rank]){ + int numids = femmodel->nodes->common_send[rank]; + for(int i=0;inodes->common_send_ids[rank][i]; + Node* node=xDynamicCast(femmodel->nodes->GetObjectByOffset(master_lid)); + _assert_(!node->IsClone()); + for(int j=0;jgsize;j++) buffer[i*maxdofspernode+j]=local_ug[node->gdoflist_local[j]]; + } + ISSM_MPI_Send(buffer,numids*maxdofspernode,ISSM_MPI_DOUBLE,rank,0,IssmComm::GetComm()); + } + } + for(int rank=0;ranknodes->common_recv[rank]){ + int numids = femmodel->nodes->common_recv[rank]; + ISSM_MPI_Recv(buffer,numids*maxdofspernode,ISSM_MPI_DOUBLE,rank,0,IssmComm::GetComm(),&status); + for(int i=0;inodes->common_recv_ids[rank][i]; + Node* node=xDynamicCast(femmodel->nodes->GetObjectByOffset(master_lid)); + for(int j=0;jgsize;j++) local_ug[node->gdoflist_local[j]] = buffer[i*maxdofspernode+j]; + } + } + } + xDelete(buffer); + + /*Now update inputs (analysis specific)*/ for(int i=0;ielements->Size();i++){ Element* element=xDynamicCast(femmodel->elements->GetObjectByOffset(i)); - analysis->InputUpdateFromSolution(solution,element); + analysis->InputUpdateFromSolution(local_ug,element); } + + /*cleanup and return*/ delete analysis; + xDelete(local_ug); } Index: /issm/trunk-jpl/src/c/modules/InputUpdateFromSolutionx/InputUpdateFromSolutionx.h =================================================================== --- /issm/trunk-jpl/src/c/modules/InputUpdateFromSolutionx/InputUpdateFromSolutionx.h (revision 23628) +++ /issm/trunk-jpl/src/c/modules/InputUpdateFromSolutionx/InputUpdateFromSolutionx.h (revision 23629) @@ -11,5 +11,4 @@ /* local prototypes: */ void InputUpdateFromSolutionx(FemModel* femmodel,Vector* solution); -void InputUpdateFromSolutionx(FemModel* femmodel,IssmDouble* solution); #endif /* _UPDATEINPUTSFROMSOLUTIONXX_H */ Index: /issm/trunk-jpl/src/c/modules/Mergesolutionfromftogx/Mergesolutionfromftogx.cpp =================================================================== --- /issm/trunk-jpl/src/c/modules/Mergesolutionfromftogx/Mergesolutionfromftogx.cpp (revision 23628) +++ /issm/trunk-jpl/src/c/modules/Mergesolutionfromftogx/Mergesolutionfromftogx.cpp (revision 23629) @@ -13,4 +13,5 @@ /*first, get gsize, fsize and ssize: */ int gsize=nodes->NumberOfDofs(GsetEnum); + int gsize_local=nodes->NumberOfDofsLocal(GsetEnum); int fsize=nodes->NumberOfDofs(FsetEnum); int ssize=nodes->NumberOfDofs(SsetEnum); @@ -29,5 +30,5 @@ /*initialize ug: */ - Vector* ug=new Vector(gsize); + Vector* ug=new Vector(gsize_local,gsize); /*Let nodes figure it out*/ Index: /issm/trunk-jpl/src/c/modules/SetActiveNodesLSMx/SetActiveNodesLSMx.cpp =================================================================== --- /issm/trunk-jpl/src/c/modules/SetActiveNodesLSMx/SetActiveNodesLSMx.cpp (revision 23628) +++ /issm/trunk-jpl/src/c/modules/SetActiveNodesLSMx/SetActiveNodesLSMx.cpp (revision 23629) @@ -85,7 +85,8 @@ /*Create vector on gset*/ - int gsize=femmodel->nodes->NumberOfDofs(GsetEnum); + int gsize = femmodel->nodes->NumberOfDofs(GsetEnum); + int glocalsize_masters = femmodel->nodes->NumberOfDofsLocal(GsetEnum); if(gsize==0) return; - Vector* vec_mask_ice=new Vector(gsize); + Vector* vec_mask_ice=new Vector(glocalsize_masters,gsize); /*Fill vector with values: */ @@ -107,12 +108,64 @@ /*Assemble vector and serialize */ vec_mask_ice->Assemble(); - IssmDouble* mask_ice=vec_mask_ice->ToMPISerial(); + + + /*FIXME: What follows should be copied and pasted into InputUpdateFromSolution Nodes*/ + + /*recover my_rank:*/ + ISSM_MPI_Status status; + int my_rank = IssmComm::GetRank(); + int num_procs = IssmComm::GetSize(); + + /*retrieve node info*/ + int glocalsize = femmodel->nodes->NumberOfDofsLocalAll(GsetEnum); + int maxdofspernode = femmodel->nodes->MaxNumDofs(GsetEnum); + + /*Get local vector of ug*/ + int *indices_ug_masters = NULL; + IssmDouble *local_ug_masters = NULL; + vec_mask_ice->GetLocalVector(&local_ug_masters,&indices_ug_masters); + _assert_(glocalsize_masters==indices_ug_masters[glocalsize_masters-1] - indices_ug_masters[0]+1); + xDelete(indices_ug_masters); delete vec_mask_ice; + + /*Now, extend vectors to account for clones (make vectors longer, for clones at the end)*/ + IssmDouble *local_ug = xNew(glocalsize); + xMemCpy(local_ug,local_ug_masters,glocalsize_masters); + xDelete(local_ug_masters); + + /*Now send and receive ug for nodes on partition edge*/ + IssmDouble* buffer = xNew(femmodel->nodes->Size()*maxdofspernode); //only one alloc + for(int rank=0;ranknodes->common_send[rank]){ + int numids = femmodel->nodes->common_send[rank]; + for(int i=0;inodes->common_send_ids[rank][i]; + Node* node=xDynamicCast(femmodel->nodes->GetObjectByOffset(master_lid)); + _assert_(!node->IsClone()); + for(int j=0;jgsize;j++) buffer[i*maxdofspernode+j]=local_ug[node->gdoflist_local[j]]; + } + ISSM_MPI_Send(buffer,numids*maxdofspernode,ISSM_MPI_DOUBLE,rank,0,IssmComm::GetComm()); + } + } + for(int rank=0;ranknodes->common_recv[rank]){ + int numids = femmodel->nodes->common_recv[rank]; + ISSM_MPI_Recv(buffer,numids*maxdofspernode,ISSM_MPI_DOUBLE,rank,0,IssmComm::GetComm(),&status); + for(int i=0;inodes->common_recv_ids[rank][i]; + Node* node=xDynamicCast(femmodel->nodes->GetObjectByOffset(master_lid)); + for(int j=0;jgsize;j++) local_ug[node->gdoflist_local[j]] = buffer[i*maxdofspernode+j]; + } + } + } + xDelete(buffer); + + /*Now update inputs (analysis specific)*/ for(int i=0;ielements->Size();i++){ Element* element=xDynamicCast(femmodel->elements->GetObjectByOffset(i)); - element->InputUpdateFromSolutionOneDof(mask_ice,IceMaskNodeActivationEnum); + element->InputUpdateFromSolutionOneDof(local_ug,IceMaskNodeActivationEnum); } - /*Clean up and return*/ - xDelete(mask_ice); + /*cleanup and return*/ + xDelete(local_ug); }/*}}}*/