Index: /issm/trunk-jpl/src/c/classes/Node.cpp
===================================================================
--- /issm/trunk-jpl/src/c/classes/Node.cpp	(revision 23626)
+++ /issm/trunk-jpl/src/c/classes/Node.cpp	(revision 23627)
@@ -18,19 +18,21 @@
 /*Node constructors and destructors:*/
 Node::Node(){/*{{{*/
-	this->approximation=0;
-
-	this->gsize    = -1;
-	this->fsize    = -1;
-	this->ssize    = -1;
-	this->clone    = false;
-	this->active   = true;
-	this->freeze   = false;
-	this->f_set    = NULL;
-	this->s_set    = NULL;
-	this->svalues  = NULL;
-	this->doftype  = NULL;
-	this->gdoflist = NULL;
-	this->fdoflist = NULL;
-	this->sdoflist = NULL;
+	this->approximation  = 0;
+	this->gsize          = -1;
+	this->fsize          = -1;
+	this->ssize          = -1;
+	this->clone          = false;
+	this->active         = true;
+	this->freeze         = false;
+	this->f_set          = NULL;
+	this->s_set          = NULL;
+	this->svalues        = NULL;
+	this->doftype        = NULL;
+	this->gdoflist       = NULL;
+	this->fdoflist       = NULL;
+	this->sdoflist       = NULL;
+	this->gdoflist_local = NULL;
+	this->fdoflist_local = NULL;
+	this->sdoflist_local = NULL;
 }
 /*}}}*/
@@ -44,4 +46,6 @@
 	this->analysis_enum = node_analysis;
 	this->clone         = node_clone;
+	this->active        = true;
+	this->freeze        = false;
 
 	/*Initialize coord_system: Identity matrix by default*/
@@ -49,27 +53,39 @@
 	for(int k=0;k<3;k++) this->coord_system[k][k]=1.0;
 
-	this->gsize    = -1;
-	this->fsize    = -1;
-	this->ssize    = -1;
-	this->active   = true;
-	this->freeze   = false;
-	this->f_set    = NULL;
-	this->s_set    = NULL;
-	this->svalues  = NULL;
-	this->doftype  = NULL;
-	this->gdoflist = NULL;
-	this->fdoflist = NULL;
-	this->sdoflist = NULL;
+	this->approximation=0;
+	if(analysis_enum==StressbalanceAnalysisEnum) this->approximation=in_approximation;
 
 	/*indexing:*/
 	this->indexingupdate = true;
+	this->doftype        = NULL;
 	Analysis *analysis = EnumToAnalysis(analysis_enum);
-	int      *doftypes = NULL;
 	this->gsize = analysis->DofsPerNode(&this->doftype,iomodel->domaintype,in_approximation);
+	delete analysis;
+
 	if(this->gsize>0){
-		this->f_set    = xNew<bool>(this->gsize);
-		this->s_set    = xNew<bool>(this->gsize);
-		this->svalues  = xNew<IssmDouble>(this->gsize);
-		this->gdoflist = xNew<int>(this->gsize);
+		this->f_set          = xNew<bool>(this->gsize);
+		this->s_set          = xNew<bool>(this->gsize);
+		this->svalues        = xNew<IssmDouble>(this->gsize);
+		this->gdoflist       = xNew<int>(this->gsize);
+		this->gdoflist_local = xNew<int>(this->gsize);
+		this->fsize          = -1;
+		this->ssize          = -1;
+		this->fdoflist       = NULL;
+		this->sdoflist       = NULL;
+		this->fdoflist_local = NULL;
+		this->sdoflist_local = NULL;
+	}
+	else{
+		this->f_set          = NULL;
+		this->s_set          = NULL;
+		this->svalues        = NULL;
+		this->gdoflist       = NULL;
+		this->gdoflist_local = NULL;
+		this->fsize          = -1;
+		this->ssize          = -1;
+		this->fdoflist       = NULL;
+		this->sdoflist       = NULL;
+		this->fdoflist_local = NULL;
+		this->sdoflist_local = NULL;
 	}
 
@@ -81,10 +97,4 @@
 		this->gdoflist[i] = -1;
 	}
-	delete analysis;
-
-	if(analysis_enum==StressbalanceAnalysisEnum)
-	 this->approximation=in_approximation;
-	else
-	 this->approximation=0;
 
 	/*Stop here if AMR*/
@@ -136,5 +146,5 @@
 				analysis_enum==HydrologyDCEfficientAnalysisEnum ||
 				analysis_enum==LevelsetAnalysisEnum
-				){
+	  ){
 		if(iomodel->domaintype!=Domain2DhorizontalEnum & iomodel->domaintype!=Domain3DsurfaceEnum){
 			/*On a 3d mesh, we may have collapsed elements, hence dead nodes. Freeze them out: */
@@ -147,5 +157,5 @@
 	if(
 				analysis_enum==FreeSurfaceTopAnalysisEnum
-				){
+	  ){
 		if(iomodel->domaintype!=Domain2DhorizontalEnum){
 			/*On a 3d mesh, we may have collapsed elements, hence dead nodes. Freeze them out: */
@@ -161,11 +171,14 @@
 Node::~Node(){/*{{{*/
 
-	if(this->f_set) xDelete<bool>(f_set); 
-	if(this->s_set) xDelete<bool>(s_set); 
-	if(this->svalues) xDelete<IssmDouble>(svalues);
-	if(this->doftype) xDelete<int>(doftype); 
-	if(this->gdoflist) xDelete<int>(gdoflist);
-	if(this->fdoflist) xDelete<int>(fdoflist);
-	if(this->sdoflist) xDelete<int>(sdoflist);
+	if(this->f_set)          xDelete<bool>(f_set);
+	if(this->s_set)          xDelete<bool>(s_set);
+	if(this->svalues)        xDelete<IssmDouble>(svalues);
+	if(this->doftype)        xDelete<int>(doftype);
+	if(this->gdoflist)       xDelete<int>(gdoflist);
+	if(this->fdoflist)       xDelete<int>(fdoflist);
+	if(this->sdoflist)       xDelete<int>(sdoflist);
+	if(this->gdoflist_local) xDelete<int>(gdoflist_local);
+	if(this->fdoflist_local) xDelete<int>(fdoflist_local);
+	if(this->sdoflist_local) xDelete<int>(sdoflist_local);
 	return;
 }
@@ -204,7 +217,14 @@
 		if(this->doftype) output->doftype=xNew<int>(output->gsize);
 		output->gdoflist=xNew<int>(output->gsize);
-	}
-	if(output->fsize>0)output->fdoflist=xNew<int>(output->fsize);
-	if(output->ssize>0)output->sdoflist=xNew<int>(output->ssize);
+		output->gdoflist_local=xNew<int>(output->gsize);
+	}
+	if(output->fsize>0){
+		output->fdoflist=xNew<int>(output->fsize);
+		output->fdoflist_local=xNew<int>(output->fsize);
+	}
+	if(output->ssize>0){
+		output->sdoflist=xNew<int>(output->ssize);
+		output->sdoflist_local=xNew<int>(output->ssize);
+	}
 
 	if(output->gsize>0){
@@ -214,7 +234,14 @@
 		if(output->doftype)memcpy(output->doftype,this->doftype,output->gsize*sizeof(int));
 		memcpy(output->gdoflist,this->gdoflist,output->gsize*sizeof(int));
-	}
-	if(output->fsize>0)memcpy(output->fdoflist,this->fdoflist,output->fsize*sizeof(int));
-	if(output->ssize>0)memcpy(output->sdoflist,this->sdoflist,output->ssize*sizeof(int));
+		memcpy(output->gdoflist_local,this->gdoflist_local,output->gsize*sizeof(int));
+	}
+	if(output->fsize>0){
+		memcpy(output->fdoflist,this->fdoflist,output->fsize*sizeof(int));
+		memcpy(output->fdoflist_local,this->fdoflist_local,output->fsize*sizeof(int));
+	}
+	if(output->ssize>0){
+		memcpy(output->sdoflist,this->sdoflist,output->ssize*sizeof(int));
+		memcpy(output->sdoflist_local,this->sdoflist_local,output->ssize*sizeof(int));
+	}
 
 	return (Object*)output; 
@@ -245,4 +272,7 @@
 	MARSHALLING_DYNAMIC(fdoflist,int,fsize);
 	MARSHALLING_DYNAMIC(sdoflist,int,ssize);
+	MARSHALLING_DYNAMIC(gdoflist_local,int,gsize);
+	MARSHALLING_DYNAMIC(fdoflist_local,int,fsize);
+	MARSHALLING_DYNAMIC(sdoflist_local,int,ssize);
 } /*}}}*/
 
@@ -284,19 +314,22 @@
 
 	_printf_("   g_doflist (" << this->gsize << "): |");
-	for(i=0;i<this->gsize;i++){
-		_printf_(" " << gdoflist[i] << " |");
-	}
+	for(i=0;i<this->gsize;i++) _printf_(" " << gdoflist[i] << " |");
 	_printf_("\n");
+	_printf_("   g_doflist_local (" << this->gsize << "): |");
+	for(i=0;i<this->gsize;i++) _printf_(" " << gdoflist_local[i] << " |");
+	_printf_("\n");
 
 	_printf_("   f_doflist (" << this->fsize << "): |");
-	for(i=0;i<this->fsize;i++){
-		_printf_(" " << fdoflist[i] << " |");
-	}
+	for(i=0;i<this->fsize;i++) _printf_(" " << fdoflist[i] << " |");
 	_printf_("\n");
+	_printf_("   f_doflist_local (" << this->fsize << "): |");
+	for(i=0;i<this->fsize;i++) _printf_(" " << fdoflist_local[i] << " |");
+	_printf_("\n");
 
 	_printf_("   s_doflist (" << this->ssize << "): |");
-	for(i=0;i<this->ssize;i++){
-		_printf_(" " << sdoflist[i] << " |");
-	}
+	for(i=0;i<this->ssize;i++) _printf_(" " << sdoflist[i] << " |");
+	_printf_("\n");
+	_printf_("   s_doflist_local (" << this->ssize << "): |");
+	for(i=0;i<this->ssize;i++) _printf_(" " << sdoflist_local[i] << " |");
 	_printf_("\n");
 
@@ -826,54 +859,61 @@
 
 /* indexing routines:*/
-void Node::DistributeDofs(int* pdofcount,int setenum){/*{{{*/
-
-	int i;
-	int dofcount;
-
-	dofcount=*pdofcount;
+void Node::AllocateDofLists(int setenum){/*{{{*/
 
 	/*Initialize: */
 	int size=0;
+
 	if(setenum==FsetEnum){
-		for(i=0;i<this->gsize;i++) if(f_set[i])size++;
+		for(int i=0;i<this->gsize;i++) if(f_set[i])size++;
 		this->fsize=size;
 		xDelete<int>(this->fdoflist);
-
-		if(this->fsize)
-		 this->fdoflist=xNew<int>(size);
-		else
-		 this->fdoflist=NULL;
-	}
-	if(setenum==SsetEnum){
-		for(i=0;i<this->gsize;i++) if(s_set[i])size++;
+		xDelete<int>(this->fdoflist_local);
+
+		if(this->fsize){
+			this->fdoflist=xNew<int>(size);
+			this->fdoflist_local=xNew<int>(size);
+		}
+		else{
+			this->fdoflist=NULL;
+			this->fdoflist_local=NULL;
+		}
+	}
+	else if(setenum==SsetEnum){
+		for(int i=0;i<this->gsize;i++) if(s_set[i])size++;
 		this->ssize=size;
 		xDelete<int>(this->sdoflist);
-
-		if(this->ssize)
-		 this->sdoflist=xNew<int>(size);
-		else
-		 this->sdoflist=NULL;
-	}
-
-	/*For clone nodfs, don't distribute dofs, we will get them from another cpu in UpdateCloneDofs!*/
-	if(clone) return;
+		xDelete<int>(this->sdoflist_local);
+
+		if(this->ssize){
+			this->sdoflist=xNew<int>(size);
+			this->sdoflist_local=xNew<int>(size);
+		}
+		else{
+			this->sdoflist=NULL;
+			this->sdoflist_local=NULL;
+		}
+	}
+	/*TODO, we should never be here for GSET! add an assert and track down whether this happens*/
+}
+/*}}}*/
+void Node::DistributeLocalDofs(int* pdofcount,int setenum){/*{{{*/
+
+	/*Get current count*/
+	int dofcount=*pdofcount;
 
 	/*This node should distribute dofs for setenum set (eg, f_set or s_set), go ahead: */
 	if(setenum==GsetEnum){
-		for(i=0;i<this->gsize;i++){
-			gdoflist[i]=dofcount+i;
-		}
+		_assert_(this->gsize==0 || this->gdoflist_local);
+		for(int i=0;i<this->gsize;i++) gdoflist_local[i]=dofcount+i;
 		dofcount+=this->gsize;
 	}
 	else if(setenum==FsetEnum){
-		for(i=0;i<this->fsize;i++){
-			fdoflist[i]=dofcount+i;
-		}
+		_assert_(this->fsize==0 || this->fdoflist_local);
+		for(int i=0;i<this->fsize;i++) fdoflist_local[i]=dofcount+i;
 		dofcount+=this->fsize;
 	}
 	else if(setenum==SsetEnum){
-		for(i=0;i<this->ssize;i++){
-			sdoflist[i]=dofcount+i;
-		}
+		_assert_(this->ssize==0 || this->sdoflist_local);
+		for(int i=0;i<this->ssize;i++) sdoflist_local[i]=dofcount+i;
 		dofcount+=this->ssize;
 	}
@@ -884,27 +924,26 @@
 }
 /*}}}*/
-void Node::OffsetDofs(int dofcount,int setenum){/*{{{*/
-
-	int i;
-
-	if(clone){
-		/*This node is a clone, don't off_set the dofs!: */
-		return;
-	}
+void Node::DistributeGlobalDofsMasters(int dofcount,int setenum){/*{{{*/
+
+	/*This node is a clone, don't offset the dofs!: */
+	if(clone) return;
 
 	/*This node should off_set the dofs, go ahead: */
 	if(setenum==GsetEnum){
-		for(i=0;i<this->gsize;i++) gdoflist[i]+=dofcount;
+		_assert_(this->gsize==0 || this->gdoflist);
+		for(int i=0;i<this->gsize;i++) this->gdoflist[i]=this->gdoflist_local[i]+dofcount;
 	}
 	else if(setenum==FsetEnum){
-		for(i=0;i<this->fsize;i++) fdoflist[i]+=dofcount;
+		_assert_(this->fsize==0 || this->fdoflist);
+		for(int i=0;i<this->fsize;i++) this->fdoflist[i]=this->fdoflist_local[i]+dofcount;
 	}
 	else if(setenum==SsetEnum){
-		for(i=0;i<this->ssize;i++) sdoflist[i]+=dofcount;
+		_assert_(this->ssize==0 || this->sdoflist);
+		for(int i=0;i<this->ssize;i++) this->sdoflist[i]=this->sdoflist_local[i]+dofcount;
 	}
 	else _error_("set of enum type " << EnumToStringx(setenum) << " not supported yet!");
 }
 /*}}}*/
-void Node::ShowTrueDofs(int* truedofs,int setenum){/*{{{*/
+void Node::ShowMasterDofs(int* truedofs,int setenum){/*{{{*/
 
 	_assert_(!this->clone);
Index: /issm/trunk-jpl/src/c/classes/Node.h
===================================================================
--- /issm/trunk-jpl/src/c/classes/Node.h	(revision 23626)
+++ /issm/trunk-jpl/src/c/classes/Node.h	(revision 23627)
@@ -24,14 +24,13 @@
 
 	private:
-		int approximation; //For ice flow models, we need to know what ice flow approximation is employed on this node
+		int  approximation; //For ice flow models, we need to know what ice flow approximation is employed on this node
+		bool clone;  //this node is replicated from another one
+
+		int  id;    // unique arbitrary id.
+		int  sid;   // "serial" id (rank of this node if the dataset was serial on 1 cpu)
+		int  lid;   // "local"  id (rank of this node in current partition)
+		int  pid;   // parallel id (specific to this partition)
 
 	public: 
-
-		int id;    // unique arbitrary id.
-		int sid;   // "serial" id (rank of this node if the dataset was serial on 1 cpu)
-		int lid;   // "local"  id (rank of this node in current partition)
-		int pid;   // parallel id (specific to this partition)
-
-		bool clone;  //this node is replicated from another one
 		int  analysis_enum;
 		IssmDouble   coord_system[3][3];
@@ -56,7 +55,10 @@
 
 		/*list of degrees of freedom: */
-		int *gdoflist;   //dof list in g_set
-		int *fdoflist;   //dof list in f_set
-		int *sdoflist;   //dof list in s_set
+		int *gdoflist;
+		int *fdoflist;
+		int *sdoflist;
+		int *gdoflist_local;
+		int *fdoflist_local;
+		int *sdoflist_local;
 
 		/*Node constructors, destructors*/
@@ -74,9 +76,10 @@
 
 		/*Node numerical routines*/
+		void  AllocateDofLists(int setenum);
 		void  Activate(void);
 		void  ApplyConstraint(int dof,IssmDouble value);
 		void  CreateNodalConstraints(Vector<IssmDouble>* ys);
 		void  Deactivate(void);
-		void  DistributeDofs(int* pdofcount,int setenum);
+		void  DistributeLocalDofs(int* pdofcount,int setenum);
 		void  DofInFSet(int dof);
 		void  DofInSSet(int dof);
@@ -94,10 +97,10 @@
 		int   IsGrounded();
 		int   Lid(void); 
-		void  OffsetDofs(int dofcount,int setenum);
+		void  DistributeGlobalDofsMasters(int dofcount,int setenum);
 		void  ReindexingDone(void);
 		void  RelaxConstraint(int dof);
 		bool  RequiresDofReindexing(void);
 		void  SetCurrentConfiguration(DataSet* nodes,Vertices* vertices);
-		void  ShowTrueDofs(int* truerows,int setenum);
+		void  ShowMasterDofs(int* truerows,int setenum);
 		int   Sid(void); 
 		void  UpdateCloneDofs(int* alltruerows,int setenum);
Index: /issm/trunk-jpl/src/c/classes/Nodes.cpp
===================================================================
--- /issm/trunk-jpl/src/c/classes/Nodes.cpp	(revision 23626)
+++ /issm/trunk-jpl/src/c/classes/Nodes.cpp	(revision 23627)
@@ -138,14 +138,26 @@
 	int num_procs = IssmComm::GetSize();
 
-	/*Go through objects, and distribute dofs locally, from 0 to numberofdofsperobject*/
+	/*First, allocate dof lists based on fset and sset*/
+	for(int i=0;i<this->Size();i++){
+		Node* node=xDynamicCast<Node*>(this->GetObjectByOffset(i));
+		node->AllocateDofLists(setenum);
+	}
+
+	/*Now, Build local dofs for masters first*/
 	int  dofcount=0;
 	for(int i=0;i<this->Size();i++){
 		Node* node=xDynamicCast<Node*>(this->GetObjectByOffset(i));
-		node->DistributeDofs(&dofcount,setenum);
+		if(!node->IsClone()) node->DistributeLocalDofs(&dofcount,setenum);
+	}
+	/*Build local dofs for clones, they always will be at the end*/
+	int dofcount_local = dofcount;
+	for(int i=0;i<this->Size();i++){
+		Node* node=xDynamicCast<Node*>(this->GetObjectByOffset(i));
+		if(node->IsClone()) node->DistributeLocalDofs(&dofcount_local,setenum);
 	}
 
 	/* Now every object has distributed dofs, but locally, and with a dof count starting from 
 	 * 0. This means the dofs between all the cpus are not unique. We now offset the dofs of each
-	 * cpus by the total last dofs of the previus cpu, starting from 0.
+	 * cpus by the total last (master) dofs of the previus cpu, starting from 0.
 	 * First: get number of dofs for each cpu*/
 	int* alldofcount=xNew<int>(num_procs);
@@ -161,5 +173,5 @@
 		/*Check that this node corresponds to our analysis currently being carried out: */
 		Node* node=xDynamicCast<Node*>(this->GetObjectByOffset(i));
-		node->OffsetDofs(offset,setenum);
+		node->DistributeGlobalDofsMasters(offset,setenum);
 	}
 
@@ -174,5 +186,5 @@
 			for(int i=0;i<numids;i++){
 				Node* node=xDynamicCast<Node*>(this->GetObjectByOffset(this->common_send_ids[rank][i]));
-				node->ShowTrueDofs(&truedofs[i*maxdofspernode+0],setenum);
+				node->ShowMasterDofs(&truedofs[i*maxdofspernode+0],setenum);
 			}
 			ISSM_MPI_Send(truedofs,numids*maxdofspernode,ISSM_MPI_INT,rank,0,IssmComm::GetComm());