Changeset 19191

Timestamp:

03/09/15 12:45:28 (10 years ago)

Author:

Eric.Larour

Message:

CHG: introducing checkpointing capability. Right now, just empty shell, getting thematlab and python apis
to accept restart_frequency settings.

Location:

issm/trunk-jpl/src

Files:

• c/classes/FemModel.cpp (modified) (7 diffs)
• c/classes/FemModel.h (modified) (3 diffs)
• c/cores/ProcessArguments.cpp (modified) (6 diffs)
• c/cores/cores.h (modified) (1 diff)
• c/cores/transient_core.cpp (modified) (3 diffs)
• c/modules/ModelProcessorx/CreateParameters.cpp (modified) (1 diff)
• c/shared/Enum/EnumDefinitions.h (modified) (2 diffs)
• c/shared/Enum/EnumToStringx.cpp (modified) (2 diffs)
• c/shared/Enum/StringToEnumx.cpp (modified) (7 diffs)
• m/classes/settings.m (modified) (5 diffs)
• m/classes/settings.py (modified) (5 diffs)
• m/enum/EnumDefinitions.py (modified) (2 diffs)
• m/enum/RestartFileNameEnum.m (added)
• m/enum/SettingsRestartFrequencyEnum.m (added)

issm/trunk-jpl/src/c/classes/FemModel.cpp ¶

@@ -54 +54 @@
         char *outbinfilename = NULL;
         char *petscfilename  = NULL;
+        char *restartfilename  = NULL;
         char *rootpath       = NULL;
 
@@ -71 +72 @@
 
         /*From command line arguments, retrieve different filenames needed to create the FemModel: */
-        ProcessArguments(&solution_type,&binfilename,&outbinfilename,&petscfilename,&lockfilename,&rootpath,argc,argv);
+        ProcessArguments(&solution_type,&binfilename,&outbinfilename,&petscfilename,&lockfilename,&restartfilename,&rootpath,argc,argv);
 
         /*Create femmodel from input files: */
         profiler->Tag(StartInit);
-        this->InitFromFiles(rootpath,binfilename,outbinfilename,petscfilename,lockfilename,solution_type,trace,NULL);
+        this->InitFromFiles(rootpath,binfilename,outbinfilename,petscfilename,lockfilename,restartfilename, solution_type,trace,NULL);
         profiler->Tag(FinishInit);
 
@@ -87 +88 @@
 }
 /*}}}*/
-FemModel::FemModel(char* rootpath, char* inputfilename, char* outputfilename, char* toolkitsfilename, char* lockfilename, ISSM_MPI_Comm incomm, int solution_type,IssmPDouble* X){ /*{{{*/
+FemModel::FemModel(char* rootpath, char* inputfilename, char* outputfilename, char* toolkitsfilename, char* lockfilename, char* restartfilename, ISSM_MPI_Comm incomm, int solution_type,IssmPDouble* X){ /*{{{*/
 
         bool traceon=true;
@@ -98 +99 @@
 
         /*Create femmodel from input files, with trace activated: */
-        this->InitFromFiles(rootpath,inputfilename,outputfilename,toolkitsfilename,lockfilename,solution_type,traceon,X);
+        this->InitFromFiles(rootpath,inputfilename,outputfilename,toolkitsfilename,lockfilename,restartfilename, solution_type,traceon,X);
 
 }
@@ -142 +143 @@
 }
 /*}}}*/
-void FemModel::InitFromFiles(char* rootpath, char* inputfilename, char* outputfilename, char* toolkitsfilename, char* lockfilename, const int in_solution_type,bool trace,IssmPDouble* X){/*{{{*/
+void FemModel::InitFromFiles(char* rootpath, char* inputfilename, char* outputfilename, char* toolkitsfilename, char* lockfilename, char* restartfilename, const int in_solution_type,bool trace,IssmPDouble* X){/*{{{*/
 
         /*intermediary*/
@@ -208 +209 @@
         this->parameters->AddObject(new StringParam(OutputFileNameEnum,outputfilename));
         this->parameters->AddObject(new StringParam(LockFileNameEnum,lockfilename));
+        this->parameters->AddObject(new StringParam(RestartFileNameEnum,restartfilename));
 
         /*Clean up*/
@@ -603 +605 @@
         else              xDelete<int>(analyses);
 }/*}}}*/
+void FemModel::CheckPoint(void){/*{{{*/
+
+        FILE* restartfid=NULL;
+        char* restartfilename = NULL;
+
+        /*First, recover the name of the restart file: */
+        parameters->FindParam(&restartfilename,RestartFileNameEnum);
+        
+        /*Open file for writing: */
+        restartfid=pfopen(restartfilename,"wb");
+
+        /*Done, close file :*/
+        pfclose(restartfid,restartfilename);
+
+        /*Free ressources: */
+        xDelete<char>(restartfilename);
+
+}
+/*}}}*/
+void FemModel::Restart(void){/*{{{*/
+}
+/*}}}*/
 
 /*Modules:*/

issm/trunk-jpl/src/c/classes/FemModel.h ¶

@@ -47 +47 @@
                 /*constructors, destructors: */
                 FemModel(int argc,char** argv,ISSM_MPI_Comm comm_init,bool trace=false);
-                FemModel(char* rootpath, char* inputfilename, char* outputfilename, char* toolkitsfilename, char* lockfilename, ISSM_MPI_Comm incomm, int solution_type,IssmPDouble* X);
+                FemModel(char* rootpath, char* inputfilename, char* outputfilename, char* toolkitsfilename, char* lockfilename, char* restartfilename, ISSM_MPI_Comm incomm, int solution_type,IssmPDouble* X);
                 ~FemModel();
 
@@ -53 +53 @@
                 void Echo();
                 FemModel* copy();
-                void InitFromFiles(char* rootpath, char* inputfilename, char* outputfilename, char* petscfilename, char* lockfilename, const int solution_type,bool trace,IssmPDouble* X=NULL);
+                void InitFromFiles(char* rootpath, char* inputfilename, char* outputfilename, char* petscfilename, char* lockfilename, char* restartfilename, const int solution_type,bool trace,IssmPDouble* X=NULL);
                 void SolutionAnalysesList(int** panalyses,int* pnumanalyses,IoModel* iomodel,int solutiontype);
                 void CleanUp(void);
@@ -60 +60 @@
                 void SetCurrentConfiguration(int configuration_type);
                 void SetCurrentConfiguration(int configuration_type,int analysis_type);
+                void CheckPoint(void);
+                void Restart(void);
 
                 /*Modules*/ 

issm/trunk-jpl/src/c/cores/ProcessArguments.cpp ¶

@@ -8 +8 @@
 #include "../shared/shared.h"
 
-void ProcessArguments(int* solution_type,char** pbinfilename,char** poutbinfilename,char** ptoolkitsfilename,char** plockfilename,char** prootpath, int argc,char **argv){
+void ProcessArguments(int* solution_type,char** pbinfilename,char** poutbinfilename,char** ptoolkitsfilename,char** plockfilename,char** prestartfilename, char** prootpath, int argc,char **argv){
 
         char *modelname      = NULL;
@@ -15 +15 @@
         char *toolkitsfilename  = NULL;
         char *lockfilename   = NULL;
+        char *restartfilename    = NULL;
         char *rootpath       = NULL;
-        char *rootpatharg    = NULL;
+        char *rootpatharg    = NULL; 
+        int   my_rank, rank_length;  
 
         /*Check input arguments*/
@@ -22 +24 @@
         if(argc<3)_error_("Usage error: missing execution directory");
         if(argc<4)_error_("Usage error: missing model name");
+
+        /*Recover myrank: */
+        my_rank=IssmComm::GetRank();
+        rank_length= (my_rank == 0 ? 1 : (int)(log10(my_rank)+1)); /*length of string "my_rank" */
 
         /*Get requested solution*/
@@ -40 +46 @@
                 toolkitsfilename  = xNew<char>(strlen(rootpath)+strlen(modelname)+strlen(".toolkits") +1); sprintf(toolkitsfilename, "%s%s%s",rootpath,modelname,".toolkits");
                 lockfilename   = xNew<char>(strlen(rootpath)+strlen(modelname)+strlen(".lock")  +1); sprintf(lockfilename,  "%s%s%s",rootpath,modelname,".lock");
+                restartfilename   = xNew<char>(strlen(rootpath)+strlen(modelname)+strlen(".rst.")  +rank_length +1); sprintf(restartfilename,  "%s%s%s%i",rootpath,modelname,".rst.",my_rank);
         }
         else{
@@ -46 +53 @@
                 toolkitsfilename  = xNew<char>(strlen(modelname)+strlen(".toolkits") +1); sprintf(toolkitsfilename, "%s%s",modelname,".toolkits");
                 lockfilename   = xNew<char>(strlen(modelname)+strlen(".lock")  +1); sprintf(lockfilename,  "%s%s",modelname,".lock");
+                restartfilename   = xNew<char>(strlen(modelname)+strlen(".rst.")  +rank_length +1); sprintf(restartfilename,  "%s%s%i",modelname,".rst.",my_rank);
         }
 
@@ -53 +61 @@
         *ptoolkitsfilename=toolkitsfilename;
         *plockfilename=lockfilename;
+        *prestartfilename=restartfilename;
         *prootpath=rootpath;
 

issm/trunk-jpl/src/c/cores/cores.h ¶

@@ -53 +53 @@
 
 //diverse
-void ProcessArguments(int* solution,char** pbinname,char** poutbinname,char** ptoolkitsname,char** plockname,char** prootpath,int argc,char **argv);
+void ProcessArguments(int* solution,char** pbinname,char** poutbinname,char** ptoolkitsname,char** plockname,char** prestartname, char** prootpath,int argc,char **argv);
 void WriteLockFile(char* filename);
 void ResetBoundaryConditions(FemModel* femmodel, int analysis_type);

issm/trunk-jpl/src/c/cores/transient_core.cpp ¶

@@ -25 +25 @@
         bool       time_adapt;
         int        output_frequency;
+        int        restart_frequency;
         int        domaintype,groundingline_migration,smb_model;
         int        numoutputs;
@@ -42 +43 @@
         femmodel->parameters->FindParam(&dakota_analysis,QmuIsdakotaEnum);
         femmodel->parameters->FindParam(&output_frequency,SettingsOutputFrequencyEnum);
+        femmodel->parameters->FindParam(&restart_frequency,SettingsRestartFrequencyEnum);
         femmodel->parameters->FindParam(&time_adapt,TimesteppingTimeAdaptEnum);
         femmodel->parameters->FindParam(&isstressbalance,TransientIsstressbalanceEnum);
@@ -179 +181 @@
                         OutputResultsx(femmodel);
                 }
+                if(step%restart_frequency==0 )femmodel->CheckPoint();
         }
 

issm/trunk-jpl/src/c/modules/ModelProcessorx/CreateParameters.cpp ¶

@@ -48 +48 @@
         parameters->AddObject(iomodel->CopyConstantObject(DomainDimensionEnum));
         parameters->AddObject(iomodel->CopyConstantObject(SettingsOutputFrequencyEnum));
+        parameters->AddObject(iomodel->CopyConstantObject(SettingsRestartFrequencyEnum));
         parameters->AddObject(iomodel->CopyConstantObject(ConstantsYtsEnum));
         parameters->AddObject(iomodel->CopyConstantObject(TimesteppingStartTimeEnum));

issm/trunk-jpl/src/c/shared/Enum/EnumDefinitions.h ¶

@@ -290 +290 @@
         SettingsLowmemEnum,
         SettingsOutputFrequencyEnum,
+        SettingsRestartFrequencyEnum,
         SettingsWaitonlockEnum,
         DebugProfilingEnum,
@@ -869 +870 @@
         InputFileNameEnum,
         LockFileNameEnum,
+        RestartFileNameEnum,
         ToolkitsOptionsAnalysesEnum,
         ToolkitsOptionsStringsEnum,

issm/trunk-jpl/src/c/shared/Enum/EnumToStringx.cpp ¶

@@ -298 +298 @@
                 case SettingsLowmemEnum : return "SettingsLowmem";
                 case SettingsOutputFrequencyEnum : return "SettingsOutputFrequency";
+                case SettingsRestartFrequencyEnum : return "SettingsRestartFrequency";
                 case SettingsWaitonlockEnum : return "SettingsWaitonlock";
                 case DebugProfilingEnum : return "DebugProfiling";
@@ -842 +843 @@
                 case InputFileNameEnum : return "InputFileName";
                 case LockFileNameEnum : return "LockFileName";
+                case RestartFileNameEnum : return "RestartFileName";
                 case ToolkitsOptionsAnalysesEnum : return "ToolkitsOptionsAnalyses";
                 case ToolkitsOptionsStringsEnum : return "ToolkitsOptionsStrings";

issm/trunk-jpl/src/c/shared/Enum/StringToEnumx.cpp ¶

@@ -304 +304 @@
               else if (strcmp(name,"SettingsLowmem")==0) return SettingsLowmemEnum;
               else if (strcmp(name,"SettingsOutputFrequency")==0) return SettingsOutputFrequencyEnum;
+              else if (strcmp(name,"SettingsRestartFrequency")==0) return SettingsRestartFrequencyEnum;
               else if (strcmp(name,"SettingsWaitonlock")==0) return SettingsWaitonlockEnum;
               else if (strcmp(name,"DebugProfiling")==0) return DebugProfilingEnum;
@@ -382 +383 @@
               else if (strcmp(name,"SurfaceforcingsBNeg")==0) return SurfaceforcingsBNegEnum;
               else if (strcmp(name,"SMBhenning")==0) return SMBhenningEnum;
-              else if (strcmp(name,"SMBcomponents")==0) return SMBcomponentsEnum;
          else stage=4;
    }
    if(stage==4){
-              if (strcmp(name,"SurfaceforcingsAccumulation")==0) return SurfaceforcingsAccumulationEnum;
+              if (strcmp(name,"SMBcomponents")==0) return SMBcomponentsEnum;
+              else if (strcmp(name,"SurfaceforcingsAccumulation")==0) return SurfaceforcingsAccumulationEnum;
               else if (strcmp(name,"SurfaceforcingsEvaporation")==0) return SurfaceforcingsEvaporationEnum;
               else if (strcmp(name,"SurfaceforcingsRunoff")==0) return SurfaceforcingsRunoffEnum;
@@ -505 +506 @@
               else if (strcmp(name,"Pengrid")==0) return PengridEnum;
               else if (strcmp(name,"Penpair")==0) return PenpairEnum;
-              else if (strcmp(name,"Profiler")==0) return ProfilerEnum;
          else stage=5;
    }
    if(stage==5){
-              if (strcmp(name,"MatrixParam")==0) return MatrixParamEnum;
+              if (strcmp(name,"Profiler")==0) return ProfilerEnum;
+              else if (strcmp(name,"MatrixParam")==0) return MatrixParamEnum;
               else if (strcmp(name,"Masscon")==0) return MassconEnum;
               else if (strcmp(name,"MassconName")==0) return MassconNameEnum;
@@ -628 +629 @@
               else if (strcmp(name,"DeviatoricStress")==0) return DeviatoricStressEnum;
               else if (strcmp(name,"DeviatoricStressxx")==0) return DeviatoricStressxxEnum;
-              else if (strcmp(name,"DeviatoricStressxy")==0) return DeviatoricStressxyEnum;
          else stage=6;
    }
    if(stage==6){
-              if (strcmp(name,"DeviatoricStressxz")==0) return DeviatoricStressxzEnum;
+              if (strcmp(name,"DeviatoricStressxy")==0) return DeviatoricStressxyEnum;
+              else if (strcmp(name,"DeviatoricStressxz")==0) return DeviatoricStressxzEnum;
               else if (strcmp(name,"DeviatoricStressyy")==0) return DeviatoricStressyyEnum;
               else if (strcmp(name,"DeviatoricStressyz")==0) return DeviatoricStressyzEnum;
@@ -751 +752 @@
               else if (strcmp(name,"Outputdefinition69")==0) return Outputdefinition69Enum;
               else if (strcmp(name,"Outputdefinition70")==0) return Outputdefinition70Enum;
-              else if (strcmp(name,"Outputdefinition71")==0) return Outputdefinition71Enum;
          else stage=7;
    }
    if(stage==7){
-              if (strcmp(name,"Outputdefinition72")==0) return Outputdefinition72Enum;
+              if (strcmp(name,"Outputdefinition71")==0) return Outputdefinition71Enum;
+              else if (strcmp(name,"Outputdefinition72")==0) return Outputdefinition72Enum;
               else if (strcmp(name,"Outputdefinition73")==0) return Outputdefinition73Enum;
               else if (strcmp(name,"Outputdefinition74")==0) return Outputdefinition74Enum;
@@ -860 +861 @@
               else if (strcmp(name,"InputFileName")==0) return InputFileNameEnum;
               else if (strcmp(name,"LockFileName")==0) return LockFileNameEnum;
+              else if (strcmp(name,"RestartFileName")==0) return RestartFileNameEnum;
               else if (strcmp(name,"ToolkitsOptionsAnalyses")==0) return ToolkitsOptionsAnalysesEnum;
               else if (strcmp(name,"ToolkitsOptionsStrings")==0) return ToolkitsOptionsStringsEnum;
@@ -873 +875 @@
               else if (strcmp(name,"BilinearInterp")==0) return BilinearInterpEnum;
               else if (strcmp(name,"NearestInterp")==0) return NearestInterpEnum;
-              else if (strcmp(name,"XY")==0) return XYEnum;
-              else if (strcmp(name,"XYZ")==0) return XYZEnum;
          else stage=8;
    }
    if(stage==8){
-              if (strcmp(name,"Dense")==0) return DenseEnum;
+              if (strcmp(name,"XY")==0) return XYEnum;
+              else if (strcmp(name,"XYZ")==0) return XYZEnum;
+              else if (strcmp(name,"Dense")==0) return DenseEnum;
               else if (strcmp(name,"MpiDense")==0) return MpiDenseEnum;
               else if (strcmp(name,"MpiSparse")==0) return MpiSparseEnum;

issm/trunk-jpl/src/m/classes/settings.m ¶

@@ -10 +10 @@
                 lowmem              = 0;
                 output_frequency    = 0;
+                restart_frequency   = 0;
                 waitonlock          = 0;
                 upload_server       = '';
@@ -58 +59 @@
                         self.output_frequency=1;
 
+                        %checkpoints frequency, by default never: 
+                        self.restart_frequency=0;
+
                         %this option can be activated to load automatically the results
                         %onto the model after a parallel run by waiting for the lock file
@@ -74 +78 @@
                         md = checkfield(md,'fieldname','settings.lowmem','numel',[1],'values',[0 1]);
                         md = checkfield(md,'fieldname','settings.output_frequency','numel',[1],'>=',1);
+                        md = checkfield(md,'fieldname','settings.restart_frequency','numel',[1],'>=',0);
                         md = checkfield(md,'fieldname','settings.waitonlock','numel',[1]);
 
@@ -84 +89 @@
                         fielddisplay(self,'lowmem','is the memory limited ? (0 or 1)');
                         fielddisplay(self,'output_frequency','frequency at which results are saved in all solutions with multiple time_steps');
+                        fielddisplay(self,'restart_frequency','frequency at which the runs are being checkpointed, allowing for a restart');
                         fielddisplay(self,'waitonlock','maximum number of minutes to wait for batch results (NaN to deactivate)');
                         fielddisplay(self,'upload_server','server hostname where model should be uploaded');
@@ -97 +103 @@
                         WriteData(fid,'object',self,'fieldname','lowmem','format','Boolean');
                         WriteData(fid,'object',self,'fieldname','output_frequency','format','Integer');
+                        WriteData(fid,'object',self,'fieldname','restart_frequency','format','Integer');
                         if self.waitonlock>0,
                                 WriteData(fid,'enum',SettingsWaitonlockEnum(),'data',true,'format','Boolean');

issm/trunk-jpl/src/m/classes/settings.py ¶

@@ -17 +17 @@
                 self.lowmem              = 0
                 self.output_frequency    = 0
+                self.restart_frequency    = 0
                 self.waitonlock          = 0
 
@@ -30 +31 @@
                 string="%s\n%s"%(string,fielddisplay(self,"lowmem","is the memory limited ? (0 or 1)"))
                 string="%s\n%s"%(string,fielddisplay(self,"output_frequency","frequency at which results are saved in all solutions with multiple time_steps"))
+                string="%s\n%s"%(string,fielddisplay(self,"restart_frequency","frequency at which the runs are being checkpointed, allowing for a restart"))
                 string="%s\n%s"%(string,fielddisplay(self,"waitonlock","maximum number of minutes to wait for batch results, or return 0"))
                 return string
@@ -44 +46 @@
                 self.output_frequency=1
 
+                #checkpoints frequency, by default never: 
+                self.restart_frequency=0
+
+
                 #this option can be activated to load automatically the results
                 #onto the model after a parallel run by waiting for the lock file
@@ -57 +63 @@
                 md = checkfield(md,'fieldname','settings.lowmem','numel',[1],'values',[0,1])
                 md = checkfield(md,'fieldname','settings.output_frequency','numel',[1],'>=',1)
+                md = checkfield(md,'fieldname','settings.restart_frequency','numel',[1],'>=',0)
                 md = checkfield(md,'fieldname','settings.waitonlock','numel',[1])
 
@@ -66 +73 @@
                 WriteData(fid,'object',self,'fieldname','lowmem','format','Boolean')
                 WriteData(fid,'object',self,'fieldname','output_frequency','format','Integer')
+                WriteData(fid,'object',self,'fieldname','restart_frequency','format','Integer')
                 if self.waitonlock>0:
                         WriteData(fid,'enum',SettingsWaitonlockEnum(),'data',True,'format','Boolean');

issm/trunk-jpl/src/m/enum/EnumDefinitions.py ¶

@@ -290 +290 @@
 def SettingsLowmemEnum(): return StringToEnum("SettingsLowmem")[0]
 def SettingsOutputFrequencyEnum(): return StringToEnum("SettingsOutputFrequency")[0]
+def SettingsRestartFrequencyEnum(): return StringToEnum("SettingsRestartFrequency")[0]
 def SettingsWaitonlockEnum(): return StringToEnum("SettingsWaitonlock")[0]
 def DebugProfilingEnum(): return StringToEnum("DebugProfiling")[0]
@@ -834 +835 @@
 def InputFileNameEnum(): return StringToEnum("InputFileName")[0]
 def LockFileNameEnum(): return StringToEnum("LockFileName")[0]
+def RestartFileNameEnum(): return StringToEnum("RestartFileName")[0]
 def ToolkitsOptionsAnalysesEnum(): return StringToEnum("ToolkitsOptionsAnalyses")[0]
 def ToolkitsOptionsStringsEnum(): return StringToEnum("ToolkitsOptionsStrings")[0]