Changeset 5903

Timestamp:

09/20/10 10:13:52 (14 years ago)

Author:

Eric.Larour

Message:

New way of running the petsc solver, that provides more flexibilty.
We have a new petscoptions field in the model, that can be filled
with petsc options, instead of the solver_string, which disappers.
The petscoptins gets resolved into a petscrc string (for serial runs)
and a petsc.rc file (for parallel runs) at runtime. The petscrc string
acts the same way than the solver_string. But the petsc.rc file acts
in a different way. Itt is loaded at runtime, when first doing Petsc Initialize.
This allows tweaking and rerunning of the same deck, each time using different
options.

Location:

issm/trunk

Files:

• src/c/EnumDefinitions/EnumDefinitions.h (modified) (1 diff)
• src/c/EnumDefinitions/EnumToString.cpp (modified) (1 diff)
• src/c/EnumDefinitions/StringToEnum.cpp (modified) (1 diff)
• src/c/modules/ModelProcessorx/CreateParameters.cpp (modified) (1 diff)
• src/c/modules/Solverx/Solverx.cpp (modified) (5 diffs)
• src/c/modules/SystemMatricesx/SystemMatricesx.cpp (modified) (1 diff)
• src/c/objects/IoModel.cpp (modified) (3 diffs)
• src/c/objects/IoModel.h (modified) (1 diff)
• src/c/solutions/issm.cpp (modified) (1 diff)
• src/m/classes/@model/model.m (modified) (1 diff)
• src/m/classes/@model/setdefaultparameters.m (modified) (1 diff)
• src/m/enum/EnumToString.m (modified) (1 diff)
• src/m/enum/PetscRcEnum.m (added)
• src/m/enum/SolverStringEnum.m (deleted)
• src/m/enum/StringToEnum.m (modified) (1 diff)
• src/m/model/PetscOptions2PetscRc.m (added)
• src/m/model/marshall.m (modified) (1 diff)
• src/m/model/presolve.m (modified) (1 diff)
• src/m/model/queue/LaunchQueueJob.m (modified) (1 diff)
• src/m/model/solvers/solversettoasm.m (modified) (1 diff)
• src/m/model/solvers/solversettomatlab.m (modified) (1 diff)
• src/m/model/solvers/solversettomumps.m (modified) (1 diff)
• src/m/model/solvers/solversettosor.m (modified) (1 diff)
• src/mex/Solver/Solver.cpp (modified) (3 diffs)
• test/NightlyRun/test104.m (modified) (1 diff)

issm/trunk/src/c/EnumDefinitions/EnumDefinitions.h ¶

@@ -365 +365 @@
         NodalEnum,
         ResponseDescriptorsEnum,
-        SolverStringEnum,
+        PetscRcEnum,
         SparsityEnum,
         TolXEnum,

issm/trunk/src/c/EnumDefinitions/EnumToString.cpp ¶

@@ -327 +327 @@
                 case NodalEnum : return "Nodal";
                 case ResponseDescriptorsEnum : return "ResponseDescriptors";
-                case SolverStringEnum : return "SolverString";
+                case PetscRcEnum : return "PetscRc";
                 case SparsityEnum : return "Sparsity";
                 case TolXEnum : return "TolX";

issm/trunk/src/c/EnumDefinitions/StringToEnum.cpp ¶

@@ -325 +325 @@
         else if (strcmp(name,"Nodal")==0) return NodalEnum;
         else if (strcmp(name,"ResponseDescriptors")==0) return ResponseDescriptorsEnum;
-        else if (strcmp(name,"SolverString")==0) return SolverStringEnum;
+        else if (strcmp(name,"PetscRc")==0) return PetscRcEnum;
         else if (strcmp(name,"Sparsity")==0) return SparsityEnum;
         else if (strcmp(name,"TolX")==0) return TolXEnum;

issm/trunk/src/c/modules/ModelProcessorx/CreateParameters.cpp ¶

@@ -64 +64 @@
         parameters->AddObject(new DoubleParam(ViscosityOvershootEnum,iomodel->viscosity_overshoot));
         parameters->AddObject(new BoolParam(WaitOnLockEnum,iomodel->waitonlock));
-        parameters->AddObject(new StringParam(SolverStringEnum,iomodel->solverstring));
+        parameters->AddObject(new StringParam(PetscRcEnum,iomodel->petscrc));
         parameters->AddObject(new IntParam(NumberOfElementsEnum,iomodel->numberofelements));
         parameters->AddObject(new BoolParam(KffEnum,iomodel->kff));

issm/trunk/src/c/modules/Solverx/Solverx.cpp ¶

@@ -27 +27 @@
         PetscTruth flag;
         int solver_type;
-        char* solver_string=NULL;
+        char* petscrc=NULL;
+        bool  fromlocalsize=true;
+
 
         /*Display message*/
@@ -44 +46 @@
         }
         else{
-                MatGetLocalSize(Kff,&local_m,&local_n);uf=NewVec(local_n,true);
+                MatGetLocalSize(Kff,&local_m,&local_n);uf=NewVec(local_n,fromlocalsize);
         }
 
-        /*Before preparing the solver, add options to the options database*/
-        parameters->FindParam(&solver_string,SolverStringEnum);
-        PetscOptionsInsertMultipleString(solver_string);
 
-        /*Process solver_string to see if we are not using special types of external solvers: */
-        PetscOptionsDetermineSolverType(&solver_type,solver_string);
+        /*Process petscrc to see if we are not using special types of external solvers: */
+        parameters->FindParam(&petscrc,PetscRcEnum);
+        PetscOptionsDetermineSolverType(&solver_type,petscrc);
+        
+        /*In serial mode, we don't have a petsc.rc file to boot the Petsc options. Do it now 
+         * using the petscrc string recover in the parameters: */
+        #ifdef _SERIAL_
+                PetscOptionsInsertMultipleString(petscrc);
 
-        #if _PETSC_VERSION_ == 2 
-        if (solver_type==MUMPSPACKAGE_LU){
-                /*Convert Kff to MATTAIJMUMPS: */
-                MatConvert(Kff,MATAIJMUMPS,MAT_REUSE_MATRIX,&Kff);
-        }
-        if (solver_type==MUMPSPACKAGE_CHOL){
-                /*Convert Kff to MATTSBAIJMUMPS: */
-                MatConvert(Kff,MATSBAIJMUMPS,MAT_REUSE_MATRIX,&Kff);
-        }
-        if (solver_type==SPOOLESPACKAGE_LU){
-                /*Convert Kff to MATTSBAIJMUMPS: */
-                MatConvert(Kff,MATAIJSPOOLES,MAT_REUSE_MATRIX,&Kff);
-        }
-        if (solver_type==SPOOLESPACKAGE_CHOL){
-                /*Convert Kff to MATTSBAIJMUMPS: */
-                MatConvert(Kff,MATSBAIJSPOOLES,MAT_REUSE_MATRIX,&Kff);
-        }
-        if (solver_type==SUPERLUDISTPACKAGE){
-                /*Convert Kff to MATTSBAIJMUMPS: */
-                MatConvert(Kff,MATSUPERLU_DIST,MAT_REUSE_MATRIX,&Kff);
-        }
+                /*In serial mode, matrices have been loaded at MPIAIJ or AIJ matrices. 
+                 * We need to convert them if we are going to run the solvers successfully: */
+                #if _PETSC_VERSION_ == 2 
+                        if (solver_type==MUMPSPACKAGE_LU){
+                                /*Convert Kff to MATTAIJMUMPS: */
+                                MatConvert(Kff,MATAIJMUMPS,MAT_REUSE_MATRIX,&Kff);
+                        }
+                        if (solver_type==MUMPSPACKAGE_CHOL){
+                                /*Convert Kff to MATTSBAIJMUMPS: */
+                                MatConvert(Kff,MATSBAIJMUMPS,MAT_REUSE_MATRIX,&Kff);
+                        }
+                        if (solver_type==SPOOLESPACKAGE_LU){
+                                /*Convert Kff to MATTSBAIJMUMPS: */
+                                MatConvert(Kff,MATAIJSPOOLES,MAT_REUSE_MATRIX,&Kff);
+                        }
+                        if (solver_type==SPOOLESPACKAGE_CHOL){
+                                /*Convert Kff to MATTSBAIJMUMPS: */
+                                MatConvert(Kff,MATSBAIJSPOOLES,MAT_REUSE_MATRIX,&Kff);
+                        }
+                        if (solver_type==SUPERLUDISTPACKAGE){
+                                /*Convert Kff to MATTSBAIJMUMPS: */
+                                MatConvert(Kff,MATSUPERLU_DIST,MAT_REUSE_MATRIX,&Kff);
+                        }
+                #endif
         #endif
 
@@ -82 +90 @@
         KSPSetFromOptions(ksp);
 
-        #if _PETSC_VERSION_ == 3 
-        /*specific solver?: */
-        KSPGetPC(ksp,&pc);
-        if (solver_type==MUMPSPACKAGE_LU){
-                PCFactorSetMatSolverPackage(pc,MAT_SOLVER_MUMPS);
-        }
+        #ifdef _SERIAL_
+                #if _PETSC_VERSION_ == 3 
+                /*specific solver?: */
+                KSPGetPC(ksp,&pc);
+                if (solver_type==MUMPSPACKAGE_LU){
+                        PCFactorSetMatSolverPackage(pc,MAT_SOLVER_MUMPS);
+                }
+                #endif
         #endif
 
@@ -98 +108 @@
                 }
         }
+
+        /*Solve: */
         KSPSolve(ksp,pf,uf);
         
@@ -106 +118 @@
         /*Free ressources:*/
         KSPFree(&ksp);
-        xfree((void**)&solver_string);
+        xfree((void**)&petscrc);
         
         /*Assign output pointers:*/

issm/trunk/src/c/modules/SystemMatricesx/SystemMatricesx.cpp ¶

@@ -160 +160 @@
                 }
         }
-        
+
         /*Assign output pointers: */
         if(pKgg) *pKgg=Kgg;

issm/trunk/src/c/objects/IoModel.cpp ¶

@@ -48 +48 @@
                 xfree((void**)&this->gridonpattyn);
         }
-        xfree((void**)&this->solverstring);
+        xfree((void**)&this->petscrc);
         xfree((void**)&this->elementonbed);
         xfree((void**)&this->elementonsurface);
@@ -182 +182 @@
         IoModelFetchData(&this->connectivity,iomodel_handle,"connectivity");
         IoModelFetchData(&this->lowmem,iomodel_handle,"lowmem");
-        IoModelFetchData(&this->solverstring,iomodel_handle,"solverstring");
+        IoModelFetchData(&this->petscrc,iomodel_handle,"petscrc");
         IoModelFetchData(&this->viscosity_overshoot,iomodel_handle,"viscosity_overshoot");
         IoModelFetchData(&this->artdiff,iomodel_handle,"artificial_diffusivity");
@@ -227 +227 @@
         this->control_analysis=0;
         this->control_parameter=NULL;
-        this->solverstring=NULL;
+        this->petscrc=NULL;
         this->numberofvariables=0;
         this->numvariabledescriptors=0;

issm/trunk/src/c/objects/IoModel.h ¶

@@ -22 +22 @@
                 int     qmu_analysis;
                 int     control_analysis;
-                char*   solverstring;
+                char*   petscrc;
 
                 /*2d mesh: */

issm/trunk/src/c/solutions/issm.cpp ¶

@@ -42 +42 @@
 
         /*Initialize Petsc and get start time*/
-        PetscInitialize(&argc,&argv,(char *)0,"");  
+        PetscInitialize(&argc,&argv,"petsc.rc","");  
         MPI_Barrier(MPI_COMM_WORLD); start=MPI_Wtime();
 

issm/trunk/src/m/classes/@model/model.m ¶

@@ -270 +270 @@
 
         %PETSc and MATLAB solver string
-        md.solverstring='';
-        md.solver_type='';
+        md.petscoptions=NaN;
+        md.petscrc='';
 
         %Analysis and sub_analysis

issm/trunk/src/m/classes/@model/setdefaultparameters.m ¶

@@ -250 +250 @@
 md.alloc_cleanup=1;
 
-%the string of solverstring is used directly by PETSc to solve finite element
-%systems KU=F. By default, we use MUMPS solver
+%set petsc options cell array, so that PETSC uses the correct solvers
 md=solversettomumps(md);
-
-%Ice solver: 'general' for Matlab's default solver (or 'lu' or 'sholesky')
-md.solver_type='general';
 
 %solution speed-up

issm/trunk/src/m/enum/EnumToString.m ¶

@@ -322 +322 @@
 case NodalEnum(), string='Nodal'; return
 case ResponseDescriptorsEnum(), string='ResponseDescriptors'; return
-case SolverStringEnum(), string='SolverString'; return
+case PetscRcEnum(), string='PetscRc'; return
 case SparsityEnum(), string='Sparsity'; return
 case TolXEnum(), string='TolX'; return

issm/trunk/src/m/enum/StringToEnum.m ¶

@@ -320 +320 @@
                         elseif (strcmpi(name,'Nodal')), enum=NodalEnum(); return
                         elseif (strcmpi(name,'ResponseDescriptors')), enum=ResponseDescriptorsEnum(); return
-                        elseif (strcmpi(name,'SolverString')), enum=SolverStringEnum(); return
+                        elseif (strcmpi(name,'PetscRc')), enum=PetscRcEnum(); return
                         elseif (strcmpi(name,'Sparsity')), enum=SparsityEnum(); return
                         elseif (strcmpi(name,'TolX')), enum=TolXEnum(); return

issm/trunk/src/m/model/marshall.m ¶

@@ -134 +134 @@
 WriteData(fid,md.lowmem,'Integer','lowmem');
 WriteData(fid,md.optscal,'Mat','optscal');
-WriteData(fid,md.solverstring,'String','solverstring');
+WriteData(fid,md.petscrc,'String','petscrc');
 WriteData(fid,md.viscosity_overshoot,'Scalar','viscosity_overshoot');
 WriteData(fid,md.stokesreconditioning,'Scalar','stokesreconditioning');

issm/trunk/src/m/model/presolve.m ¶

@@ -33 +33 @@
         count=count+numpairsforthisrift;
 end
+
+%deal with petsc options
+md.petscrc=PetscOptions2PetscRc(md.petscoptions,'petsc.rc');

issm/trunk/src/m/model/queue/LaunchQueueJob.m ¶

@@ -22 +22 @@
         
         %compress the files into one zip.
-        compressstring=['tar -zcf ' md.runtimename '.tar.gz ' md.name '.bin ' md.name '.queue '];
+        compressstring=['tar -zcf ' md.runtimename '.tar.gz ' md.name '.bin ' md.name '.queue petsc.rc'];
         if md.qmu_analysis,
                 compressstring=[compressstring md.name '.qmu.in'];

issm/trunk/src/m/model/solvers/solversettoasm.m ¶

@@ -5 +5 @@
 %      md=solversettoasm(md)
 
-%md.solverstring=' -mat_type aij -ksp_type cgs -pc_type asm -sub_mat_type mumps -sub_pc_type lu -pc_asm_overlap 4 -pc_factor_shift_positive_definite true';
-%md.solverstring=' -mat_type aij -ksp_type cgs -pc_type asm -sub_pc_type lu -pc_asm_overlap 4 ';
-md.solverstring=' -mat_type aij -ksp_type gmres -pc_type asm -sub_pc_type lu -pc_asm_overlap 4 ';
+%md.petscoptions={{'mat_type','aij'},{'ksp_type','cgs'},{'pc_type','asm'},{sub_mat_type','mumps'},{'sub_pc_type','lu'},{'pc_asm_overlap',4},{'pc_factor_shift_positive_definite','true'}};
+%md.petscoptions={{'mat_type','aij'},{'ksp_type','cgs'},{'pc_type','asm'},{'sub_pc_type','lu'},{'pc_asm_overlap',4}};
+md.petscoptions={{'mat_type','aij'},{'ksp_type','gmres'},{'pc_type','asm'},{'sub_pc_type','lu'},{'pc_asm_overlap',4}};

issm/trunk/src/m/model/solvers/solversettomatlab.m ¶

@@ -5 +5 @@
 %      md=solversettomatlab(md)
 
-md.solverstring='-ksp_type matlab';
+md.petscoptions={{'ksp_type','matlab'}};

issm/trunk/src/m/model/solvers/solversettomumps.m ¶

@@ -5 +5 @@
 %      md=solversettomumps(md)
 
-md.solverstring='-mat_type aijmumps -ksp_type preonly -pc_type lu -mat_mumps_icntl_14 120 -pc_factor_shift_positive_definite true';
-
-%optional
-%md.solverstring=[md.solverstring ' -mat_mumps_icntl_14 40 -mat_mumps_icntl_4 0 -mat_mumps_sym 1 -mat_ignore_lower_triangular'];
+md.petscoptions={{'mat_type','aijmumps'},{'ksp_type','preonly'},{'pc_type','lu'},{'mat_mumps_icntl_14',120},{'pc_factor_shift_positive_definite','true'}};

issm/trunk/src/m/model/solvers/solversettosor.m ¶

@@ -5 +5 @@
 %      md=solversettosor(md)
 
-md.mat_type='aij';
-md.ksp_type='cg';
-md.pc_type='sor';
-md.solver_extra_option=' -pc_sor_omega 1.1 -pc_sor_its 2 ';
+md.petscoptions={{'mat_type','aij'},{'ksp_type','cg'},{'pc_type','sor'},{'pc_sor_omega',1.1},{pc_sor_its',2}};

issm/trunk/src/mex/Solver/Solver.cpp ¶

@@ -12 +12 @@
         Vec         uf0           = NULL;
         Vec         uf            = NULL;
-        char       *solver_string = NULL;
+        char       *petscrc = NULL;
         Parameters *parameters    = NULL;
         int         verbose;
@@ -27 +27 @@
         /*First, check solver string: */
         FetchParams(&parameters,PARAMETERS);
-        parameters->FindParam(&solver_string,SolverStringEnum);
+        parameters->FindParam(&petscrc,PetscRcEnum);
         int verbose; parameters->FindParam(&verbose,VerboseEnum);
         
         /*Fetch rest of data only if not running the matlab solver: */
-        if (strcmp(solver_string,matlabstring)!=0){
+        if (strcmp(petscrc,matlabstring)!=0){
                 /*Input datasets: */
                 FetchData(&Kff,KFF);
@@ -55 +55 @@
         VecFree(&uf);
         delete parameters;
-        xfree((void**)&solver_string);
+        xfree((void**)&petscrc);
 
         MODULEEND();

issm/trunk/test/NightlyRun/test104.m ¶

@@ -5 +5 @@
 md=setelementstype(md,'macayeal','all');
 md.cluster=oshostname;
+md.verbose=1;
 md=solve(md,'analysis_type',DiagnosticSolutionEnum);