Index: /issm/trunk-jpl/src/c/classes/FemModel.cpp =================================================================== --- /issm/trunk-jpl/src/c/classes/FemModel.cpp (revision 19190) +++ /issm/trunk-jpl/src/c/classes/FemModel.cpp (revision 19191) @@ -54,4 +54,5 @@ char *outbinfilename = NULL; char *petscfilename = NULL; + char *restartfilename = NULL; char *rootpath = NULL; @@ -71,9 +72,9 @@ /*From command line arguments, retrieve different filenames needed to create the FemModel: */ - ProcessArguments(&solution_type,&binfilename,&outbinfilename,&petscfilename,&lockfilename,&rootpath,argc,argv); + ProcessArguments(&solution_type,&binfilename,&outbinfilename,&petscfilename,&lockfilename,&restartfilename,&rootpath,argc,argv); /*Create femmodel from input files: */ profiler->Tag(StartInit); - this->InitFromFiles(rootpath,binfilename,outbinfilename,petscfilename,lockfilename,solution_type,trace,NULL); + this->InitFromFiles(rootpath,binfilename,outbinfilename,petscfilename,lockfilename,restartfilename, solution_type,trace,NULL); profiler->Tag(FinishInit); @@ -87,5 +88,5 @@ } /*}}}*/ -FemModel::FemModel(char* rootpath, char* inputfilename, char* outputfilename, char* toolkitsfilename, char* lockfilename, ISSM_MPI_Comm incomm, int solution_type,IssmPDouble* X){ /*{{{*/ +FemModel::FemModel(char* rootpath, char* inputfilename, char* outputfilename, char* toolkitsfilename, char* lockfilename, char* restartfilename, ISSM_MPI_Comm incomm, int solution_type,IssmPDouble* X){ /*{{{*/ bool traceon=true; @@ -98,5 +99,5 @@ /*Create femmodel from input files, with trace activated: */ - this->InitFromFiles(rootpath,inputfilename,outputfilename,toolkitsfilename,lockfilename,solution_type,traceon,X); + this->InitFromFiles(rootpath,inputfilename,outputfilename,toolkitsfilename,lockfilename,restartfilename, solution_type,traceon,X); } @@ -142,5 +143,5 @@ } /*}}}*/ -void FemModel::InitFromFiles(char* rootpath, char* inputfilename, char* outputfilename, char* toolkitsfilename, char* lockfilename, const int in_solution_type,bool trace,IssmPDouble* X){/*{{{*/ +void FemModel::InitFromFiles(char* rootpath, char* inputfilename, char* outputfilename, char* toolkitsfilename, char* lockfilename, char* restartfilename, const int in_solution_type,bool trace,IssmPDouble* X){/*{{{*/ /*intermediary*/ @@ -208,4 +209,5 @@ this->parameters->AddObject(new StringParam(OutputFileNameEnum,outputfilename)); this->parameters->AddObject(new StringParam(LockFileNameEnum,lockfilename)); + this->parameters->AddObject(new StringParam(RestartFileNameEnum,restartfilename)); /*Clean up*/ @@ -603,4 +605,26 @@ else xDelete(analyses); }/*}}}*/ +void FemModel::CheckPoint(void){/*{{{*/ + + FILE* restartfid=NULL; + char* restartfilename = NULL; + + /*First, recover the name of the restart file: */ + parameters->FindParam(&restartfilename,RestartFileNameEnum); + + /*Open file for writing: */ + restartfid=pfopen(restartfilename,"wb"); + + /*Done, close file :*/ + pfclose(restartfid,restartfilename); + + /*Free ressources: */ + xDelete(restartfilename); + +} +/*}}}*/ +void FemModel::Restart(void){/*{{{*/ +} +/*}}}*/ /*Modules:*/ Index: /issm/trunk-jpl/src/c/classes/FemModel.h =================================================================== --- /issm/trunk-jpl/src/c/classes/FemModel.h (revision 19190) +++ /issm/trunk-jpl/src/c/classes/FemModel.h (revision 19191) @@ -47,5 +47,5 @@ /*constructors, destructors: */ FemModel(int argc,char** argv,ISSM_MPI_Comm comm_init,bool trace=false); - FemModel(char* rootpath, char* inputfilename, char* outputfilename, char* toolkitsfilename, char* lockfilename, ISSM_MPI_Comm incomm, int solution_type,IssmPDouble* X); + FemModel(char* rootpath, char* inputfilename, char* outputfilename, char* toolkitsfilename, char* lockfilename, char* restartfilename, ISSM_MPI_Comm incomm, int solution_type,IssmPDouble* X); ~FemModel(); @@ -53,5 +53,5 @@ void Echo(); FemModel* copy(); - void InitFromFiles(char* rootpath, char* inputfilename, char* outputfilename, char* petscfilename, char* lockfilename, const int solution_type,bool trace,IssmPDouble* X=NULL); + void InitFromFiles(char* rootpath, char* inputfilename, char* outputfilename, char* petscfilename, char* lockfilename, char* restartfilename, const int solution_type,bool trace,IssmPDouble* X=NULL); void SolutionAnalysesList(int** panalyses,int* pnumanalyses,IoModel* iomodel,int solutiontype); void CleanUp(void); @@ -60,4 +60,6 @@ void SetCurrentConfiguration(int configuration_type); void SetCurrentConfiguration(int configuration_type,int analysis_type); + void CheckPoint(void); + void Restart(void); /*Modules*/ Index: /issm/trunk-jpl/src/c/cores/ProcessArguments.cpp =================================================================== --- /issm/trunk-jpl/src/c/cores/ProcessArguments.cpp (revision 19190) +++ /issm/trunk-jpl/src/c/cores/ProcessArguments.cpp (revision 19191) @@ -8,5 +8,5 @@ #include "../shared/shared.h" -void ProcessArguments(int* solution_type,char** pbinfilename,char** poutbinfilename,char** ptoolkitsfilename,char** plockfilename,char** prootpath, int argc,char **argv){ +void ProcessArguments(int* solution_type,char** pbinfilename,char** poutbinfilename,char** ptoolkitsfilename,char** plockfilename,char** prestartfilename, char** prootpath, int argc,char **argv){ char *modelname = NULL; @@ -15,6 +15,8 @@ char *toolkitsfilename = NULL; char *lockfilename = NULL; + char *restartfilename = NULL; char *rootpath = NULL; - char *rootpatharg = NULL; + char *rootpatharg = NULL; + int my_rank, rank_length; /*Check input arguments*/ @@ -22,4 +24,8 @@ if(argc<3)_error_("Usage error: missing execution directory"); if(argc<4)_error_("Usage error: missing model name"); + + /*Recover myrank: */ + my_rank=IssmComm::GetRank(); + rank_length= (my_rank == 0 ? 1 : (int)(log10(my_rank)+1)); /*length of string "my_rank" */ /*Get requested solution*/ @@ -40,4 +46,5 @@ toolkitsfilename = xNew(strlen(rootpath)+strlen(modelname)+strlen(".toolkits") +1); sprintf(toolkitsfilename, "%s%s%s",rootpath,modelname,".toolkits"); lockfilename = xNew(strlen(rootpath)+strlen(modelname)+strlen(".lock") +1); sprintf(lockfilename, "%s%s%s",rootpath,modelname,".lock"); + restartfilename = xNew(strlen(rootpath)+strlen(modelname)+strlen(".rst.") +rank_length +1); sprintf(restartfilename, "%s%s%s%i",rootpath,modelname,".rst.",my_rank); } else{ @@ -46,4 +53,5 @@ toolkitsfilename = xNew(strlen(modelname)+strlen(".toolkits") +1); sprintf(toolkitsfilename, "%s%s",modelname,".toolkits"); lockfilename = xNew(strlen(modelname)+strlen(".lock") +1); sprintf(lockfilename, "%s%s",modelname,".lock"); + restartfilename = xNew(strlen(modelname)+strlen(".rst.") +rank_length +1); sprintf(restartfilename, "%s%s%i",modelname,".rst.",my_rank); } @@ -53,4 +61,5 @@ *ptoolkitsfilename=toolkitsfilename; *plockfilename=lockfilename; + *prestartfilename=restartfilename; *prootpath=rootpath; Index: /issm/trunk-jpl/src/c/cores/cores.h =================================================================== --- /issm/trunk-jpl/src/c/cores/cores.h (revision 19190) +++ /issm/trunk-jpl/src/c/cores/cores.h (revision 19191) @@ -53,5 +53,5 @@ //diverse -void ProcessArguments(int* solution,char** pbinname,char** poutbinname,char** ptoolkitsname,char** plockname,char** prootpath,int argc,char **argv); +void ProcessArguments(int* solution,char** pbinname,char** poutbinname,char** ptoolkitsname,char** plockname,char** prestartname, char** prootpath,int argc,char **argv); void WriteLockFile(char* filename); void ResetBoundaryConditions(FemModel* femmodel, int analysis_type); Index: /issm/trunk-jpl/src/c/cores/transient_core.cpp =================================================================== --- /issm/trunk-jpl/src/c/cores/transient_core.cpp (revision 19190) +++ /issm/trunk-jpl/src/c/cores/transient_core.cpp (revision 19191) @@ -25,4 +25,5 @@ bool time_adapt; int output_frequency; + int restart_frequency; int domaintype,groundingline_migration,smb_model; int numoutputs; @@ -42,4 +43,5 @@ femmodel->parameters->FindParam(&dakota_analysis,QmuIsdakotaEnum); femmodel->parameters->FindParam(&output_frequency,SettingsOutputFrequencyEnum); + femmodel->parameters->FindParam(&restart_frequency,SettingsRestartFrequencyEnum); femmodel->parameters->FindParam(&time_adapt,TimesteppingTimeAdaptEnum); femmodel->parameters->FindParam(&isstressbalance,TransientIsstressbalanceEnum); @@ -179,4 +181,5 @@ OutputResultsx(femmodel); } + if(step%restart_frequency==0 )femmodel->CheckPoint(); } Index: /issm/trunk-jpl/src/c/modules/ModelProcessorx/CreateParameters.cpp =================================================================== --- /issm/trunk-jpl/src/c/modules/ModelProcessorx/CreateParameters.cpp (revision 19190) +++ /issm/trunk-jpl/src/c/modules/ModelProcessorx/CreateParameters.cpp (revision 19191) @@ -48,4 +48,5 @@ parameters->AddObject(iomodel->CopyConstantObject(DomainDimensionEnum)); parameters->AddObject(iomodel->CopyConstantObject(SettingsOutputFrequencyEnum)); + parameters->AddObject(iomodel->CopyConstantObject(SettingsRestartFrequencyEnum)); parameters->AddObject(iomodel->CopyConstantObject(ConstantsYtsEnum)); parameters->AddObject(iomodel->CopyConstantObject(TimesteppingStartTimeEnum)); Index: /issm/trunk-jpl/src/c/shared/Enum/EnumDefinitions.h =================================================================== --- /issm/trunk-jpl/src/c/shared/Enum/EnumDefinitions.h (revision 19190) +++ /issm/trunk-jpl/src/c/shared/Enum/EnumDefinitions.h (revision 19191) @@ -290,4 +290,5 @@ SettingsLowmemEnum, SettingsOutputFrequencyEnum, + SettingsRestartFrequencyEnum, SettingsWaitonlockEnum, DebugProfilingEnum, @@ -869,4 +870,5 @@ InputFileNameEnum, LockFileNameEnum, + RestartFileNameEnum, ToolkitsOptionsAnalysesEnum, ToolkitsOptionsStringsEnum, Index: /issm/trunk-jpl/src/c/shared/Enum/EnumToStringx.cpp =================================================================== --- /issm/trunk-jpl/src/c/shared/Enum/EnumToStringx.cpp (revision 19190) +++ /issm/trunk-jpl/src/c/shared/Enum/EnumToStringx.cpp (revision 19191) @@ -298,4 +298,5 @@ case SettingsLowmemEnum : return "SettingsLowmem"; case SettingsOutputFrequencyEnum : return "SettingsOutputFrequency"; + case SettingsRestartFrequencyEnum : return "SettingsRestartFrequency"; case SettingsWaitonlockEnum : return "SettingsWaitonlock"; case DebugProfilingEnum : return "DebugProfiling"; @@ -842,4 +843,5 @@ case InputFileNameEnum : return "InputFileName"; case LockFileNameEnum : return "LockFileName"; + case RestartFileNameEnum : return "RestartFileName"; case ToolkitsOptionsAnalysesEnum : return "ToolkitsOptionsAnalyses"; case ToolkitsOptionsStringsEnum : return "ToolkitsOptionsStrings"; Index: /issm/trunk-jpl/src/c/shared/Enum/StringToEnumx.cpp =================================================================== --- /issm/trunk-jpl/src/c/shared/Enum/StringToEnumx.cpp (revision 19190) +++ /issm/trunk-jpl/src/c/shared/Enum/StringToEnumx.cpp (revision 19191) @@ -304,4 +304,5 @@ else if (strcmp(name,"SettingsLowmem")==0) return SettingsLowmemEnum; else if (strcmp(name,"SettingsOutputFrequency")==0) return SettingsOutputFrequencyEnum; + else if (strcmp(name,"SettingsRestartFrequency")==0) return SettingsRestartFrequencyEnum; else if (strcmp(name,"SettingsWaitonlock")==0) return SettingsWaitonlockEnum; else if (strcmp(name,"DebugProfiling")==0) return DebugProfilingEnum; @@ -382,9 +383,9 @@ else if (strcmp(name,"SurfaceforcingsBNeg")==0) return SurfaceforcingsBNegEnum; else if (strcmp(name,"SMBhenning")==0) return SMBhenningEnum; - else if (strcmp(name,"SMBcomponents")==0) return SMBcomponentsEnum; else stage=4; } if(stage==4){ - if (strcmp(name,"SurfaceforcingsAccumulation")==0) return SurfaceforcingsAccumulationEnum; + if (strcmp(name,"SMBcomponents")==0) return SMBcomponentsEnum; + else if (strcmp(name,"SurfaceforcingsAccumulation")==0) return SurfaceforcingsAccumulationEnum; else if (strcmp(name,"SurfaceforcingsEvaporation")==0) return SurfaceforcingsEvaporationEnum; else if (strcmp(name,"SurfaceforcingsRunoff")==0) return SurfaceforcingsRunoffEnum; @@ -505,9 +506,9 @@ else if (strcmp(name,"Pengrid")==0) return PengridEnum; else if (strcmp(name,"Penpair")==0) return PenpairEnum; - else if (strcmp(name,"Profiler")==0) return ProfilerEnum; else stage=5; } if(stage==5){ - if (strcmp(name,"MatrixParam")==0) return MatrixParamEnum; + if (strcmp(name,"Profiler")==0) return ProfilerEnum; + else if (strcmp(name,"MatrixParam")==0) return MatrixParamEnum; else if (strcmp(name,"Masscon")==0) return MassconEnum; else if (strcmp(name,"MassconName")==0) return MassconNameEnum; @@ -628,9 +629,9 @@ else if (strcmp(name,"DeviatoricStress")==0) return DeviatoricStressEnum; else if (strcmp(name,"DeviatoricStressxx")==0) return DeviatoricStressxxEnum; - else if (strcmp(name,"DeviatoricStressxy")==0) return DeviatoricStressxyEnum; else stage=6; } if(stage==6){ - if (strcmp(name,"DeviatoricStressxz")==0) return DeviatoricStressxzEnum; + if (strcmp(name,"DeviatoricStressxy")==0) return DeviatoricStressxyEnum; + else if (strcmp(name,"DeviatoricStressxz")==0) return DeviatoricStressxzEnum; else if (strcmp(name,"DeviatoricStressyy")==0) return DeviatoricStressyyEnum; else if (strcmp(name,"DeviatoricStressyz")==0) return DeviatoricStressyzEnum; @@ -751,9 +752,9 @@ else if (strcmp(name,"Outputdefinition69")==0) return Outputdefinition69Enum; else if (strcmp(name,"Outputdefinition70")==0) return Outputdefinition70Enum; - else if (strcmp(name,"Outputdefinition71")==0) return Outputdefinition71Enum; else stage=7; } if(stage==7){ - if (strcmp(name,"Outputdefinition72")==0) return Outputdefinition72Enum; + if (strcmp(name,"Outputdefinition71")==0) return Outputdefinition71Enum; + else if (strcmp(name,"Outputdefinition72")==0) return Outputdefinition72Enum; else if (strcmp(name,"Outputdefinition73")==0) return Outputdefinition73Enum; else if (strcmp(name,"Outputdefinition74")==0) return Outputdefinition74Enum; @@ -860,4 +861,5 @@ else if (strcmp(name,"InputFileName")==0) return InputFileNameEnum; else if (strcmp(name,"LockFileName")==0) return LockFileNameEnum; + else if (strcmp(name,"RestartFileName")==0) return RestartFileNameEnum; else if (strcmp(name,"ToolkitsOptionsAnalyses")==0) return ToolkitsOptionsAnalysesEnum; else if (strcmp(name,"ToolkitsOptionsStrings")==0) return ToolkitsOptionsStringsEnum; @@ -873,10 +875,10 @@ else if (strcmp(name,"BilinearInterp")==0) return BilinearInterpEnum; else if (strcmp(name,"NearestInterp")==0) return NearestInterpEnum; - else if (strcmp(name,"XY")==0) return XYEnum; - else if (strcmp(name,"XYZ")==0) return XYZEnum; else stage=8; } if(stage==8){ - if (strcmp(name,"Dense")==0) return DenseEnum; + if (strcmp(name,"XY")==0) return XYEnum; + else if (strcmp(name,"XYZ")==0) return XYZEnum; + else if (strcmp(name,"Dense")==0) return DenseEnum; else if (strcmp(name,"MpiDense")==0) return MpiDenseEnum; else if (strcmp(name,"MpiSparse")==0) return MpiSparseEnum; Index: /issm/trunk-jpl/src/m/classes/settings.m =================================================================== --- /issm/trunk-jpl/src/m/classes/settings.m (revision 19190) +++ /issm/trunk-jpl/src/m/classes/settings.m (revision 19191) @@ -10,4 +10,5 @@ lowmem = 0; output_frequency = 0; + restart_frequency = 0; waitonlock = 0; upload_server = ''; @@ -58,4 +59,7 @@ self.output_frequency=1; + %checkpoints frequency, by default never: + self.restart_frequency=0; + %this option can be activated to load automatically the results %onto the model after a parallel run by waiting for the lock file @@ -74,4 +78,5 @@ md = checkfield(md,'fieldname','settings.lowmem','numel',[1],'values',[0 1]); md = checkfield(md,'fieldname','settings.output_frequency','numel',[1],'>=',1); + md = checkfield(md,'fieldname','settings.restart_frequency','numel',[1],'>=',0); md = checkfield(md,'fieldname','settings.waitonlock','numel',[1]); @@ -84,4 +89,5 @@ fielddisplay(self,'lowmem','is the memory limited ? (0 or 1)'); fielddisplay(self,'output_frequency','frequency at which results are saved in all solutions with multiple time_steps'); + fielddisplay(self,'restart_frequency','frequency at which the runs are being checkpointed, allowing for a restart'); fielddisplay(self,'waitonlock','maximum number of minutes to wait for batch results (NaN to deactivate)'); fielddisplay(self,'upload_server','server hostname where model should be uploaded'); @@ -97,4 +103,5 @@ WriteData(fid,'object',self,'fieldname','lowmem','format','Boolean'); WriteData(fid,'object',self,'fieldname','output_frequency','format','Integer'); + WriteData(fid,'object',self,'fieldname','restart_frequency','format','Integer'); if self.waitonlock>0, WriteData(fid,'enum',SettingsWaitonlockEnum(),'data',true,'format','Boolean'); Index: /issm/trunk-jpl/src/m/classes/settings.py =================================================================== --- /issm/trunk-jpl/src/m/classes/settings.py (revision 19190) +++ /issm/trunk-jpl/src/m/classes/settings.py (revision 19191) @@ -17,4 +17,5 @@ self.lowmem = 0 self.output_frequency = 0 + self.restart_frequency = 0 self.waitonlock = 0 @@ -30,4 +31,5 @@ string="%s\n%s"%(string,fielddisplay(self,"lowmem","is the memory limited ? (0 or 1)")) string="%s\n%s"%(string,fielddisplay(self,"output_frequency","frequency at which results are saved in all solutions with multiple time_steps")) + string="%s\n%s"%(string,fielddisplay(self,"restart_frequency","frequency at which the runs are being checkpointed, allowing for a restart")) string="%s\n%s"%(string,fielddisplay(self,"waitonlock","maximum number of minutes to wait for batch results, or return 0")) return string @@ -44,4 +46,8 @@ self.output_frequency=1 + #checkpoints frequency, by default never: + self.restart_frequency=0 + + #this option can be activated to load automatically the results #onto the model after a parallel run by waiting for the lock file @@ -57,4 +63,5 @@ md = checkfield(md,'fieldname','settings.lowmem','numel',[1],'values',[0,1]) md = checkfield(md,'fieldname','settings.output_frequency','numel',[1],'>=',1) + md = checkfield(md,'fieldname','settings.restart_frequency','numel',[1],'>=',0) md = checkfield(md,'fieldname','settings.waitonlock','numel',[1]) @@ -66,4 +73,5 @@ WriteData(fid,'object',self,'fieldname','lowmem','format','Boolean') WriteData(fid,'object',self,'fieldname','output_frequency','format','Integer') + WriteData(fid,'object',self,'fieldname','restart_frequency','format','Integer') if self.waitonlock>0: WriteData(fid,'enum',SettingsWaitonlockEnum(),'data',True,'format','Boolean'); Index: /issm/trunk-jpl/src/m/enum/EnumDefinitions.py =================================================================== --- /issm/trunk-jpl/src/m/enum/EnumDefinitions.py (revision 19190) +++ /issm/trunk-jpl/src/m/enum/EnumDefinitions.py (revision 19191) @@ -290,4 +290,5 @@ def SettingsLowmemEnum(): return StringToEnum("SettingsLowmem")[0] def SettingsOutputFrequencyEnum(): return StringToEnum("SettingsOutputFrequency")[0] +def SettingsRestartFrequencyEnum(): return StringToEnum("SettingsRestartFrequency")[0] def SettingsWaitonlockEnum(): return StringToEnum("SettingsWaitonlock")[0] def DebugProfilingEnum(): return StringToEnum("DebugProfiling")[0] @@ -834,4 +835,5 @@ def InputFileNameEnum(): return StringToEnum("InputFileName")[0] def LockFileNameEnum(): return StringToEnum("LockFileName")[0] +def RestartFileNameEnum(): return StringToEnum("RestartFileName")[0] def ToolkitsOptionsAnalysesEnum(): return StringToEnum("ToolkitsOptionsAnalyses")[0] def ToolkitsOptionsStringsEnum(): return StringToEnum("ToolkitsOptionsStrings")[0] Index: /issm/trunk-jpl/src/m/enum/RestartFileNameEnum.m =================================================================== --- /issm/trunk-jpl/src/m/enum/RestartFileNameEnum.m (revision 19191) +++ /issm/trunk-jpl/src/m/enum/RestartFileNameEnum.m (revision 19191) @@ -0,0 +1,11 @@ +function macro=RestartFileNameEnum() +%RESTARTFILENAMEENUM - Enum of RestartFileName +% +% WARNING: DO NOT MODIFY THIS FILE +% this file has been automatically generated by src/c/shared/Enum/Synchronize.sh +% Please read src/c/shared/Enum/README for more information +% +% Usage: +% macro=RestartFileNameEnum() + +macro=StringToEnum('RestartFileName'); Index: /issm/trunk-jpl/src/m/enum/SettingsRestartFrequencyEnum.m =================================================================== --- /issm/trunk-jpl/src/m/enum/SettingsRestartFrequencyEnum.m (revision 19191) +++ /issm/trunk-jpl/src/m/enum/SettingsRestartFrequencyEnum.m (revision 19191) @@ -0,0 +1,11 @@ +function macro=SettingsRestartFrequencyEnum() +%SETTINGSRESTARTFREQUENCYENUM - Enum of SettingsRestartFrequency +% +% WARNING: DO NOT MODIFY THIS FILE +% this file has been automatically generated by src/c/shared/Enum/Synchronize.sh +% Please read src/c/shared/Enum/README for more information +% +% Usage: +% macro=SettingsRestartFrequencyEnum() + +macro=StringToEnum('SettingsRestartFrequency');