Index: /issm/trunk-jpl/src/c/Makefile.am =================================================================== --- /issm/trunk-jpl/src/c/Makefile.am (revision 13538) +++ /issm/trunk-jpl/src/c/Makefile.am (revision 13539) @@ -221,4 +221,5 @@ ./shared/Elements/ComputeDelta18oTemperaturePrecipitation.cpp\ ./shared/String/sharedstring.h\ + ./shared/String/DescriptorIndex.cpp\ ./shared/Wrapper/wrappershared.h\ ./shared/Wrapper/ModuleBoot.cpp\ @@ -373,12 +374,7 @@ ./modules/AverageOntoPartitionx/AverageOntoPartitionx.cpp\ ./modules/AverageOntoPartitionx/AverageOntoPartitionx.h\ - ./modules/Dakotax/Dakotax.h\ - ./modules/Dakotax/Dakotax.cpp\ - ./modules/Dakotax/DakotaMPI_Bcast.cpp\ - ./modules/Dakotax/DakotaFree.cpp\ - ./modules/Dakotax/SpawnCore.cpp\ - ./modules/Dakotax/DescriptorIndex.cpp\ ./modules/ModelProcessorx/Dakota/CreateParametersDakota.cpp\ - ./modules/Dakotax/SpawnCoreParallel.cpp + ./solutions/dakota_core.cpp\ + ./solutions/DakotaSpawnCore.cpp #}}} #Transient sources {{{ Index: /issm/trunk-jpl/src/c/classes/dakota/DakotaPlugin.cpp =================================================================== --- /issm/trunk-jpl/src/c/classes/dakota/DakotaPlugin.cpp (revision 13538) +++ /issm/trunk-jpl/src/c/classes/dakota/DakotaPlugin.cpp (revision 13539) @@ -88,5 +88,5 @@ /*run core solution: */ - SpawnCore(responses,numFns, variables,variable_descriptors,numACV,femmodel,counter); + DakotaSpawnCore(responses,numFns, variables,variable_descriptors,numACV,femmodel,counter); /*populate responses: */ Index: /issm/trunk-jpl/src/c/classes/objects/Profiler.cpp =================================================================== --- /issm/trunk-jpl/src/c/classes/objects/Profiler.cpp (revision 13538) +++ /issm/trunk-jpl/src/c/classes/objects/Profiler.cpp (revision 13539) @@ -16,10 +16,10 @@ /*FUNCTION Profiler::Profiler() default constructor {{{*/ Profiler::Profiler(){ - this->timetags=new Parameters(); + this->time=new Parameters(); } /*}}}*/ /*FUNCTION Profiler::~Profiler(){{{*/ Profiler::~Profiler(){ - delete timetags; + delete time; } /*}}}*/ @@ -31,5 +31,5 @@ _printLine_("Profiler:"); _printLine_(" time tags: "); - this->timetags->Echo(); + this->time->Echo(); } @@ -40,5 +40,5 @@ _printLine_("Profiler:"); _printLine_(" time tags: "); - this->timetags->DeepEcho(); + this->time->DeepEcho(); } @@ -59,5 +59,7 @@ void Profiler::Tag(int tagenum,bool dontmpisync){ - double time; + IssmDouble t; + IssmDouble f; + IssmDouble m; /*If mpisync requested, make sure all the cpus are at the same point @@ -69,22 +71,32 @@ } - /*Now capture time: */ + /*Capture time: */ #ifdef _HAVE_MPI_ - time=MPI_Wtime(); + t=MPI_Wtime(); #else - time=(IssmPDouble)clock(); + t=(IssmPDouble)clock(); #endif - /*Plug into this->timetags: */ - this->timetags->AddObject(new DoubleParam(tagenum,time)); + /*Capture flops: */ + #ifdef _HAVE_PETSC_ + PetscGetFlops(&f); + PetscMemoryGetCurrentUsage(&m); + #else + /*do nothing for now:*/ + #endif + + /*Plug into this->time: */ + this->time->AddObject(new DoubleParam(tagenum,t)); + this->flops->AddObject(new DoubleParam(tagenum,f)); + this->memory->AddObject(new DoubleParam(tagenum,m)); } /*}}}*/ -/*FUNCTION Profiler::Delta {{{*/ -double Profiler::Delta(int inittag, int finaltag){ +/*FUNCTION Profiler::DeltaTime {{{*/ +IssmDouble Profiler::DeltaTime(int inittag, int finaltag){ - double init, final; - this->timetags->FindParam(&init,inittag); - this->timetags->FindParam(&final,finaltag); + IssmDouble init, final; + this->time->FindParam(&init,inittag); + this->time->FindParam(&final,finaltag); #ifdef _HAVE_MPI_ @@ -95,45 +107,64 @@ } /*}}}*/ -/*FUNCTION Profiler::DeltaModHour {{{*/ -int Profiler::DeltaModHour(int inittag, int finishtag){ +/*FUNCTION Profiler::DeltaFlops {{{*/ +IssmDouble Profiler::DeltaFlops(int inittag, int finaltag){ - double init, finish; - this->timetags->FindParam(&init,inittag); - this->timetags->FindParam(&finish,finishtag); + IssmDouble init, final; + this->flops->FindParam(&init,inittag); + this->flops->FindParam(&final,finaltag); + + return final-init; +} +/*}}}*/ +/*FUNCTION Profiler::DeltaTimeModHour {{{*/ +int Profiler::DeltaTimeModHour(int inittag, int finishtag){ + + IssmDouble init, finish; + this->time->FindParam(&init,inittag); + this->time->FindParam(&finish,finishtag); #ifdef _HAVE_MPI_ - return int((finish-init)/3600); + return int((reCast(finish-init))/3600); #else - return int((finish-init)/CLOCKS_PER_SEC/3600); + return int((reCast(finish-init))/CLOCKS_PER_SEC/3600); #endif } /*}}}*/ -/*FUNCTION Profiler::DeltaModMin {{{*/ -int Profiler::DeltaModMin(int inittag, int finishtag){ +/*FUNCTION Profiler::DeltaTimeModMin {{{*/ +int Profiler::DeltaTimeModMin(int inittag, int finishtag){ - double init, finish; - this->timetags->FindParam(&init,inittag); - this->timetags->FindParam(&finish,finishtag); + IssmDouble init, finish; + this->time->FindParam(&init,inittag); + this->time->FindParam(&finish,finishtag); #ifdef _HAVE_MPI_ - return int(int(finish-init)%3600/60); + return int(int(reCast(finish-init))%3600/60); #else - return int(int(finish-init)/CLOCKS_PER_SEC%3600/60); + return int(int(reCast(finish-init))/CLOCKS_PER_SEC%3600/60); #endif } /*}}}*/ -/*FUNCTION Profiler::DeltaModSec {{{*/ -int Profiler::DeltaModSec(int inittag, int finishtag){ +/*FUNCTION Profiler::DeltaTimeModSec {{{*/ +int Profiler::DeltaTimeModSec(int inittag, int finishtag){ - double init, finish; - this->timetags->FindParam(&init,inittag); - this->timetags->FindParam(&finish,finishtag); + IssmDouble init, finish; + this->time->FindParam(&init,inittag); + this->time->FindParam(&finish,finishtag); #ifdef _HAVE_MPI_ - return int(finish-init)%60; + return int(reCast(finish-init))%60; #else - return int(finish-init)/CLOCKS_PER_SEC%60; + return int(reCast(finish-init))/CLOCKS_PER_SEC%60; #endif } /*}}}*/ +/*FUNCTION Profiler::Memory {{{*/ +IssmDouble Profiler::Memory(int tag){ + + IssmDouble m; + this->memory->FindParam(&m,tag); + + return m; +} +/*}}}*/ Index: /issm/trunk-jpl/src/c/classes/objects/Profiler.h =================================================================== --- /issm/trunk-jpl/src/c/classes/objects/Profiler.h (revision 13538) +++ /issm/trunk-jpl/src/c/classes/objects/Profiler.h (revision 13539) @@ -28,5 +28,7 @@ public: - Parameters* timetags; + Parameters* time; + Parameters* flops; + Parameters* memory; /*Profiler constructors, destructors {{{*/ @@ -43,8 +45,10 @@ /*Profiler routines {{{*/ void Tag(int tagenum,bool dontmpisync=false); - double Delta(int inittag, int finaltag); - int DeltaModHour(int inittag, int finaltag); - int DeltaModMin(int inittag, int finaltag); - int DeltaModSec(int inittag, int finaltag); + IssmDouble Memory(int tag); + IssmDouble DeltaTime(int inittag, int finaltag); + IssmDouble DeltaFlops(int inittag, int finaltag); + int DeltaTimeModHour(int inittag, int finaltag); + int DeltaTimeModMin(int inittag, int finaltag); + int DeltaTimeModSec(int inittag, int finaltag); /*}}}*/ }; Index: sm/trunk-jpl/src/c/modules/Dakotax/DakotaFree.cpp =================================================================== --- /issm/trunk-jpl/src/c/modules/Dakotax/DakotaFree.cpp (revision 13538) +++ (revision ) @@ -1,57 +1,0 @@ -/*!\file: DakotaFree.cpp - * \brief DakotaFree: free allocations on other cpus, not done by Dakota. - - */ - -#ifdef HAVE_CONFIG_H - #include -#else -#error "Cannot compile with HAVE_CONFIG_H symbol! run configure first!" -#endif - -#include "../../Container/Container.h" -#include "../../shared/shared.h" -#include "../../include/include.h" -#include "../../EnumDefinitions/EnumDefinitions.h" -#include "../modules.h" - - -void DakotaFree(double** pvariables,char*** pvariables_descriptors,char*** presponses_descriptors,int numvariables,int numresponses){ - - int i; - extern int my_rank; - - double *variables = NULL; - char **variables_descriptors = NULL; - char **responses_descriptors = NULL; - char *string = NULL; - - /*recover pointers: */ - variables=*pvariables; - variables_descriptors=*pvariables_descriptors; - responses_descriptors=*presponses_descriptors; - - - /*Free variables and variables_descriptors only on cpu !=0*/ - if(my_rank!=0){ - xDelete(variables); - for(i=0;i(string); - } - xDelete(variables_descriptors); - } - - //responses descriptors on every cpu - for(i=0;i(string); - } - //rest of dynamic allocations. - xDelete(responses_descriptors); - - /*Assign output pointers:*/ - *pvariables=variables; - *pvariables_descriptors=variables_descriptors; - *presponses_descriptors=responses_descriptors; -} Index: sm/trunk-jpl/src/c/modules/Dakotax/DakotaMPI_Bcast.cpp =================================================================== --- /issm/trunk-jpl/src/c/modules/Dakotax/DakotaMPI_Bcast.cpp (revision 13538) +++ (revision ) @@ -1,66 +1,0 @@ -/*!\file: DakotaMPI_Bcast - * \brief: broadcast variables_descriptors, variables, numvariables and numresponses - * from cpu 0 to all other cpus. - */ - -#ifdef HAVE_CONFIG_H - #include -#else -#error "Cannot compile with HAVE_CONFIG_H symbol! run configure first!" -#endif - -#include "./Dakotax.h" - -void DakotaMPI_Bcast(double** pvariables, char*** pvariables_descriptors,int* pnumvariables, int* pnumresponses){ - - int i; - extern int my_rank; - - /*inputs and outputs: */ - double* variables=NULL; - char** variables_descriptors=NULL; - int numvariables; - int numresponses; - - /*intermediary: */ - char* string=NULL; - int string_length; - - - /*recover inputs from pointers: */ - variables=*pvariables; - variables_descriptors=*pvariables_descriptors; - numvariables=*pnumvariables; - numresponses=*pnumresponses; - - /*numvariables: */ - MPI_Bcast(&numvariables,1,MPI_INT,0,MPI_COMM_WORLD); - - /*variables:*/ - if(my_rank!=0)variables=xNew(numvariables); - MPI_Bcast(variables,numvariables,MPI_DOUBLE,0,MPI_COMM_WORLD); - - /*variables_descriptors: */ - if(my_rank!=0){ - variables_descriptors=xNew(numvariables); - } - for(i=0;i(string_length); - MPI_Bcast(string,string_length,MPI_CHAR,0,MPI_COMM_WORLD); - if(my_rank!=0)variables_descriptors[i]=string; - } - - /*numresponses: */ - MPI_Bcast(&numresponses,1,MPI_INT,0,MPI_COMM_WORLD); - - /*Assign output pointers:*/ - *pnumvariables=numvariables; - *pvariables=variables; - *pvariables_descriptors=variables_descriptors; - *pnumresponses=numresponses; -} Index: sm/trunk-jpl/src/c/modules/Dakotax/Dakotax.cpp =================================================================== --- /issm/trunk-jpl/src/c/modules/Dakotax/Dakotax.cpp (revision 13538) +++ (revision ) @@ -1,128 +1,0 @@ -/*!\file: Dakotax.cpp - * \brief: wrapper to the Dakota capabilities. qmu fires up Dakota, and registers a Dakota Pluggin - * which will be in charge of running the solution sequences repeteadly, to garner statistics. - * - * This routine deals with running ISSM and Dakota in library mode. In library mode, Dakota does not - * run as an execuatble. Its capabilities are linked into the ISSM software. ISSM calls dakota routines - * directly from the dakota library. qmu.cpp is the code that is in charge of calling those routines. - * - * Dakota has its own way of running in parallel (for embarassingly parallel jobs). We do not want that, - * as ISSM knows exactly how to run "really parallel" jobs that use all CPUS. To bypass Dakota's parallelism, - * we overloaded the constructor for the parallel library (see the Dakota patch in the externalpackages/dakota - * directory). This overloaded constructor fires up Dakota serially on CPU 0 only! We take care of broadcasting - * to the other CPUS, hence ISSM is running in parallel, and Dakota serially on CPU0. - * - * Now, how does CPU 0 drive all other CPUS to carry out sensitivity analysese? By synchronizing its call to - * our ISSM cores (diagnostic_core, thermal_core, transient_core, etc ...) on CPU 0 with all other CPUS. - * This explains the structure of qmu.cpp, where cpu 0 runs Dakota, the Dakota pluggin fires up SpawnCore.cpp, - * while the other CPUS are waiting for a broadcast from CPU0, once they get it, they also fire up - * SpawnCore. In the end, SpawnCore is fired up on all CPUS, with CPU0 having Dakota inputs, that it will - * broacast to other CPUS. - * - * Now, how does dakota call the SpawnCore routine? The SpawnCore is embedded into the DakotaPlugin object - * which is derived from the Direct Interface Dakota objct. This is the only way to run Dakota in library - * mode (see their developper guide for more info). Dakota registers the DakotaPlugin object into its own - * database, and calls on the embedded SpawnCore from CPU0. - * - */ - -#ifdef HAVE_CONFIG_H -#include -#else -#error "Cannot compile with HAVE_CONFIG_H symbol! run configure first!" -#endif - -#include "./Dakotax.h" - -#include "../../shared/shared.h" -#include "../../include/include.h" -#include "../../io/io.h" -#include "../../toolkits/toolkits.h" -#include "../../EnumDefinitions/EnumDefinitions.h" -#include "../../classes/dakota/DakotaPlugin.h" - -#ifdef _HAVE_DAKOTA_ //only works if dakota library has been compiled in. -#include "ParallelLibrary.H" -#include "ProblemDescDB.H" -#include "DakotaStrategy.H" -#include "DakotaModel.H" -#include "DakotaInterface.H" - -#endif - -void Dakotax(FemModel* femmodel){ - - - #ifdef _HAVE_DAKOTA_ //only works if dakota library has been compiled in. - - extern int my_rank; - char* dakota_input_file = NULL; - char* dakota_output_file = NULL; - char* dakota_error_file = NULL; - int status=0; - Dakota::ModelLIter ml_iter; - Parameters* parameters = NULL; - - /*Retrieve parameters: */ - parameters=femmodel->parameters; - - /*Recover dakota_input_file, dakota_output_file and dakota_error_file, in the parameters dataset in parallel */ - parameters->FindParam(&dakota_input_file,QmuInNameEnum); - parameters->FindParam(&dakota_output_file,QmuOutNameEnum); - parameters->FindParam(&dakota_error_file,QmuErrNameEnum); - - if(my_rank==0){ - - // Instantiate/initialize the parallel library and problem description - // database objects. - Dakota::ParallelLibrary parallel_lib("serial"); //use our own ISSM Dakota library mode constructor, which only fires up Dakota on CPU 0. - Dakota::ProblemDescDB problem_db(parallel_lib); - - // Manage input file parsing, output redirection, and restart processing - // without a CommandLineHandler. This version relies on parsing of an - // input file. - problem_db.manage_inputs(dakota_input_file); - // specify_outputs_restart() is only necessary if specifying non-defaults - parallel_lib.specify_outputs_restart(dakota_output_file,dakota_error_file,NULL,NULL); - - // Instantiate the Strategy object (which instantiates all Model and - // Iterator objects) using the parsed information in problem_db. - Dakota::Strategy selected_strategy(problem_db); - - // convenience function for iterating over models and performing any - // interface plug-ins - Dakota::ModelList& models = problem_db.model_list(); - - for (ml_iter = models.begin(); ml_iter != models.end(); ml_iter++) { - - Dakota::Interface& interface = ml_iter->interface(); - - //set DB nodes to the existing Model specification - problem_db.set_db_model_nodes(ml_iter->model_id()); - - // Serial case: plug in derived Interface object without an analysisComm - interface.assign_rep(new SIM::DakotaPlugin(problem_db,(void*)femmodel), false); - } - - // Execute the strategy - problem_db.lock(); // prevent run-time DB queries - selected_strategy.run_strategy(); - - //Warn other cpus that we are done running the dakota iterator, by setting the counter to -1: - SpawnCore(NULL,0, NULL,NULL,0,femmodel,-1); - - } - else{ - - for(;;){ - if(!SpawnCore(NULL,0, NULL,NULL,0,femmodel,0))break; //counter came in at -1 on cpu0, bail out. - } - } - - /*Free ressources:*/ - xDelete(dakota_input_file); - xDelete(dakota_error_file); - xDelete(dakota_output_file); - - #endif //#ifdef _HAVE_DAKOTA_ -} Index: /issm/trunk-jpl/src/c/modules/Dakotax/Dakotax.h =================================================================== --- /issm/trunk-jpl/src/c/modules/Dakotax/Dakotax.h (revision 13538) +++ /issm/trunk-jpl/src/c/modules/Dakotax/Dakotax.h (revision 13539) @@ -10,12 +10,6 @@ /* local prototypes: */ -int SpawnCore(double* responses, int numresponses, double* variables, char** variables_descriptors,int numvariables, void* femmodel,int counter); -int DescriptorIndex(char* root, int* pindex,char* descriptor); - void Dakotax(FemModel* femmodel); -void SpawnCoreParallel(double* responses, int numresponses, double* variables, char** variables_descriptors,int numvariables, FemModel* femmodel,int counter); -void DakotaResponses(double* responses,char** responses_descriptors,int numresponses,FemModel* femmodel); -void DakotaMPI_Bcast(double** pvariables, char*** pvariables_descriptors,int* pnumvariables, int* pnumresponses); -void DakotaFree(double** pvariables,char*** pvariables_descriptors,char*** presponses_descriptors,int numvariables,int numresponses); +int DakotaSpawnCore(double* responses, int numresponses, double* variables, char** variables_descriptors,int numvariables, void* femmodel,int counter); #endif /* _DAKOTAX_H */ Index: sm/trunk-jpl/src/c/modules/Dakotax/DescriptorIndex.cpp =================================================================== --- /issm/trunk-jpl/src/c/modules/Dakotax/DescriptorIndex.cpp (revision 13538) +++ (revision ) @@ -1,71 +1,0 @@ -/*!\file: DescriptorIndex: return type of qmu variable: indexed, scaled, nodal or regular - * + figure out the descriptor root. - * Ex: scaled_Thickness_1 should return SCALEDENUM, fill root with Thickness, and initialize index - * to 1. - */ - -#ifdef HAVE_CONFIG_H - #include -#else -#error "Cannot compile with HAVE_CONFIG_H symbol! run configure first!" -#endif - -#include -#include -#include -#include "../../shared/shared.h" -#include "../../EnumDefinitions/EnumDefinitions.h" - -int DescriptorIndex(char* root, int* pindex,char* descriptor){ //We assume root has already been allocated, and we just have to copy into it. - - char * pch=NULL; - - /*retrieve first token, separated by underscore: */ - pch = strtok (descriptor,"_"); - if(!pch)_error_("descriptor " << descriptor << " is not correctly formatted!"); - - if (strncmp(pch,"scaled",6)==0){ - /*we have a scaled variable. recover the root: */ - pch = strtok (NULL, "_"); - if(!pch)_error_("scaled descriptor " << descriptor << " is not correctly formatted!"); - memcpy(root,pch,(strlen(pch)+1)*sizeof(char)); - - /*now recover the index if it exists: */ - pch = strtok (NULL, "_"); - if(!pch){ - *pindex=-1; - } - else{ - sscanf(pch,"%i",pindex); - } - return ScaledEnum; - } - else if (strncmp(pch,"indexed",7)==0){ - /*we have an indexed variable. recover the root: */ - pch = strtok (NULL, "_"); - if(!pch)_error_("indexed descriptor " << descriptor << " is not correctly formatted!"); - memcpy(root,pch,(strlen(pch)+1)*sizeof(char)); - /*now recover the index: */ - pch = strtok (NULL, "_"); - if(!pch)_error_("indexed descriptor " << descriptor << " is not correctly formatted!"); - sscanf(pch,"%i",pindex); - return IndexedEnum; - } - else if (strncmp(pch,"nodal",5)==0){ - /*we have an indexed variable. recover the root: */ - pch = strtok (NULL, "_"); - if(!pch)_error_("nodal descriptor " << descriptor << " is not correctly formatted!"); - memcpy(root,pch,(strlen(pch)+1)*sizeof(char)); - /*now recover the index: */ - pch = strtok (NULL, "_"); - if(!pch)_error_("nodal descriptor " << descriptor << " is not correctly formatted!"); - sscanf(pch,"%i",pindex); - return NodalEnum; - } - else{ - /*We don't have _ in the name, this is a regular variable: */ - memcpy(root,pch,(strlen(pch)+1)*sizeof(char)); - *pindex=-1; - return RegularEnum; - } -} Index: sm/trunk-jpl/src/c/modules/Dakotax/SpawnCore.cpp =================================================================== --- /issm/trunk-jpl/src/c/modules/Dakotax/SpawnCore.cpp (revision 13538) +++ (revision ) @@ -1,29 +1,0 @@ -/*!\file: SpawnCore.cpp - * \brief: branch into SpawnCoreMatlab and SpawnCoreParallel. - */ - -#ifdef HAVE_CONFIG_H - #include -#else -#error "Cannot compile with HAVE_CONFIG_H symbol! run configure first!" -#endif - - -#include "../../classes/objects/objects.h" -#include "../../io/io.h" -#include "../../EnumDefinitions/EnumDefinitions.h" -#include "../../shared/shared.h" -#include "./Dakotax.h" -#include "../../include/include.h" - -int SpawnCore(double* responses, int numresponses, double* variables, char** variables_descriptors,int numvariables, void* femmodel,int counter){ - - /*Branch into a serial SpawnCore and a parallel SpawnCore: */ - - /*Call SpawnCoreParallel unless counter=-1 on cpu0, in which case, bail out and return 0: */ - MPI_Bcast(&counter,1,MPI_INT,0,MPI_COMM_WORLD); - if(counter==-1)return 0; - - SpawnCoreParallel(responses, numresponses, variables, variables_descriptors,numvariables, (FemModel*)femmodel,counter); - return 1; -} Index: sm/trunk-jpl/src/c/modules/Dakotax/SpawnCoreParallel.cpp =================================================================== --- /issm/trunk-jpl/src/c/modules/Dakotax/SpawnCoreParallel.cpp (revision 13538) +++ (revision ) @@ -1,87 +1,0 @@ -/*!\file: SpawnCoreParallel.cpp - * \brief: run core ISSM solution using Dakota inputs coming from CPU 0. - * \sa qmu.cpp DakotaPlugin.cpp - * - * This routine needs to be understood simultaneously with qmu.cpp and DakotaPlugin. - * SpawnCoreParallel is called by all CPUS, with CPU 0 holding Dakota variable values, along - * with variable descriptors. - * - * SpawnCoreParallel takes care of broadcasting the variables and their descriptors across the MPI - * ring. Once this is done, we use the variables to modify the inputs for the solution core. - * For ex, if "rho_ice" is provided, for ex 920, we include "rho_ice" in the inputs, then - * call the core with the modified inputs. This is the way we get Dakota to explore the parameter - * spce of the core. - * - * Once the core is called, we process the results of the core, and using the processed results, - * we compute response functions. The responses are computed on all CPUS, but they are targeted - * for CPU 0, which will get these values back to the Dakota engine. - * - */ - -#ifdef HAVE_CONFIG_H - #include -#else -#error "Cannot compile with HAVE_CONFIG_H symbol! run configure first!" -#endif - - -#include "../../classes/objects/objects.h" -#include "../../io/io.h" -#include "../../EnumDefinitions/EnumDefinitions.h" -#include "../../io/io.h" -#include "../../shared/shared.h" -#include "./Dakotax.h" -#include "../../include/include.h" -#include "../../solutions/solutions.h" -#include "../modules.h" - -void SpawnCoreParallel(double* d_responses, int d_numresponses, double* d_variables, char** d_variables_descriptors,int d_numvariables, FemModel* femmodel,int counter){ - - /*Notice the d_, which prefixes anything that is being provided to us by the Dakota pluggin. Careful. some things are ours, some are dakotas!: */ - - int i; - extern int my_rank; - - char **responses_descriptors = NULL; //these are our! there are only numresponsedescriptors of them, not d_numresponses!!! - int numresponsedescriptors; - char *string = NULL; - int string_length; - int solution_type; - bool control_analysis = false; - void (*solutioncore)(FemModel*) = NULL; - - /*synchronize all cpus, as CPU 0 is probably late (it is starting the entire dakota strategy!) : */ - MPI_Barrier(MPI_COMM_WORLD); - if(VerboseQmu()) _pprintLine_("qmu iteration: " << counter); - - /*retrieve parameters: */ - femmodel->parameters->FindParam(&responses_descriptors,&numresponsedescriptors,QmuResponsedescriptorsEnum); - femmodel->parameters->FindParam(&solution_type,SolutionTypeEnum); - femmodel->parameters->FindParam(&control_analysis,InversionIscontrolEnum); - - /* only cpu 0, running dakota is providing us with variables and variables_descriptors and numresponses: broadcast onto other cpus: */ - DakotaMPI_Bcast(&d_variables,&d_variables_descriptors,&d_numvariables,&d_numresponses); - - /*Modify core inputs in objects contained in femmodel, to reflect the dakota variables inputs: */ - InputUpdateFromDakotax(femmodel->elements,femmodel->nodes,femmodel->vertices,femmodel->loads,femmodel->materials,femmodel->parameters,d_variables,d_variables_descriptors,d_numvariables); - - /*Determine solution sequence: */ - if(VerboseQmu()) _pprintLine_("Starting " << EnumToStringx(solution_type) << " core:"); - CorePointerFromSolutionEnum(&solutioncore,femmodel->parameters,solution_type); - #ifdef _HAVE_CONTROL_ - if(control_analysis)solutioncore=&control_core; - #else - _error_("ISSM was not compiled with control capabilities, exiting!"); - #endif - - /*Run the core solution sequence: */ - solutioncore(femmodel); - - /*compute responses: */ - if(VerboseQmu()) _pprintLine_("compute dakota responses:"); - DakotaResponsesx(d_responses,femmodel->elements,femmodel->nodes,femmodel->vertices,femmodel->loads,femmodel->materials,femmodel->parameters,responses_descriptors,numresponsedescriptors,d_numresponses); - - /*Free ressources:*/ - DakotaFree(&d_variables,&d_variables_descriptors,&responses_descriptors, d_numvariables, numresponsedescriptors); -} - Index: /issm/trunk-jpl/src/c/shared/String/sharedstring.h =================================================================== --- /issm/trunk-jpl/src/c/shared/String/sharedstring.h (revision 13538) +++ /issm/trunk-jpl/src/c/shared/String/sharedstring.h (revision 13539) @@ -8,4 +8,8 @@ int DescriptorIndex(char* root, int* pindex,char* descriptor); +#ifndef WIN32 +int stricmp(const char* a,const char* b); +#endif + #endif //ifndef _SHAREDSTRING_H_ Index: /issm/trunk-jpl/src/c/shared/shared.h =================================================================== --- /issm/trunk-jpl/src/c/shared/shared.h (revision 13538) +++ /issm/trunk-jpl/src/c/shared/shared.h (revision 13539) @@ -16,4 +16,5 @@ #include "Numerics/numerics.h" #include "Sorting/sorting.h" +#include "String/sharedstring.h" #include "Threads/issm_threads.h" #include "TriMesh/trimesh.h" Index: /issm/trunk-jpl/src/c/solutions/DakotaSpawnCore.cpp =================================================================== --- /issm/trunk-jpl/src/c/solutions/DakotaSpawnCore.cpp (revision 13539) +++ /issm/trunk-jpl/src/c/solutions/DakotaSpawnCore.cpp (revision 13539) @@ -0,0 +1,198 @@ +/*!\file: DakotaSpawnCore.cpp + + * \brief: run core ISSM solution using Dakota inputs coming from CPU 0. + * \sa qmu.cpp DakotaPlugin.cpp + * + * This routine needs to be understood simultaneously with qmu.cpp and DakotaPlugin. + * DakotaSpawnCoreParallel is called by all CPUS, with CPU 0 holding Dakota variable values, along + * with variable descriptors. + * + * DakotaSpawnCoreParallel takes care of broadcasting the variables and their descriptors across the MPI + * ring. Once this is done, we use the variables to modify the inputs for the solution core. + * For ex, if "rho_ice" is provided, for ex 920, we include "rho_ice" in the inputs, then + * call the core with the modified inputs. This is the way we get Dakota to explore the parameter + * spce of the core. + * + * Once the core is called, we process the results of the core, and using the processed results, + * we compute response functions. The responses are computed on all CPUS, but they are targeted + * for CPU 0, which will get these values back to the Dakota engine. + * + */ + +/*Includes and prototypes: {{{*/ +#ifdef HAVE_CONFIG_H + #include +#else +#error "Cannot compile with HAVE_CONFIG_H symbol! run configure first!" +#endif + + +#include "../classes/objects/objects.h" +#include "../io/io.h" +#include "../EnumDefinitions/EnumDefinitions.h" +#include "../shared/shared.h" +#include "../include/include.h" +#include "../solutions/solutions.h" +#include "../modules/modules.h" + +void DakotaMPI_Bcast(double** pvariables, char*** pvariables_descriptors,int* pnumvariables, int* pnumresponses); +void DakotaFree(double** pvariables,char*** pvariables_descriptors,char*** presponses_descriptors,int numvariables,int numresponses); + +/*}}}*/ + +/*Notice the d_, which prefixes anything that is being provided to us by the Dakota pluggin. Careful. some things are ours, some are dakotas!: */ +int DakotaSpawnCore(double* d_responses, int d_numresponses, double* d_variables, char** d_variables_descriptors,int d_numvariables, void* void_femmodel,int counter){ + + int i; + extern int my_rank; + + char **responses_descriptors = NULL; //these are our! there are only numresponsedescriptors of them, not d_numresponses!!! + int numresponsedescriptors; + char *string = NULL; + int string_length; + int solution_type; + bool control_analysis = false; + void (*solutioncore)(FemModel*) = NULL; + FemModel* femmodel = NULL; + + + /*If counter==-1 on cpu0, it means that the dakota runs are done. In which case, bail out and return 0: */ + #ifdef _HAVE_MPI_ + MPI_Bcast(&counter,1,MPI_INT,0,MPI_COMM_WORLD); + #endif + if(counter==-1)return 0; + + /*cast void_femmodel to FemModel: */ + femmodel=(FemModel*)void_femmodel; + + /*retrieve parameters: */ + femmodel->parameters->FindParam(&responses_descriptors,&numresponsedescriptors,QmuResponsedescriptorsEnum); + femmodel->parameters->FindParam(&solution_type,SolutionTypeEnum); + femmodel->parameters->FindParam(&control_analysis,InversionIscontrolEnum); + + if(VerboseQmu()) _pprintLine_("qmu iteration: " << counter); + + /* only cpu 0, running dakota is providing us with variables and variables_descriptors and numresponses: broadcast onto other cpus: */ + DakotaMPI_Bcast(&d_variables,&d_variables_descriptors,&d_numvariables,&d_numresponses); + + /*Modify core inputs in objects contained in femmodel, to reflect the dakota variables inputs: */ + InputUpdateFromDakotax(femmodel->elements,femmodel->nodes,femmodel->vertices,femmodel->loads,femmodel->materials,femmodel->parameters,d_variables,d_variables_descriptors,d_numvariables); + + /*Determine solution sequence: */ + if(VerboseQmu()) _pprintLine_("Starting " << EnumToStringx(solution_type) << " core:"); + CorePointerFromSolutionEnum(&solutioncore,femmodel->parameters,solution_type); + #ifdef _HAVE_CONTROL_ + if(control_analysis)solutioncore=&control_core; + #else + _error_("ISSM was not compiled with control capabilities, exiting!"); + #endif + + /*Run the core solution sequence: */ + solutioncore(femmodel); + + /*compute responses: */ + if(VerboseQmu()) _pprintLine_("compute dakota responses:"); + DakotaResponsesx(d_responses,femmodel->elements,femmodel->nodes,femmodel->vertices,femmodel->loads,femmodel->materials,femmodel->parameters,responses_descriptors,numresponsedescriptors,d_numresponses); + + /*Free ressources:*/ + DakotaFree(&d_variables,&d_variables_descriptors,&responses_descriptors, d_numvariables, numresponsedescriptors); +} + +void DakotaMPI_Bcast(double** pvariables, char*** pvariables_descriptors,int* pnumvariables, int* pnumresponses){ /*{{{*/ + + /* * \brief: broadcast variables_descriptors, variables, numvariables and numresponses + * from cpu 0 to all other cpus. + */ + + int i; + extern int my_rank; + + /*inputs and outputs: */ + double* variables=NULL; + char** variables_descriptors=NULL; + int numvariables; + int numresponses; + + /*intermediary: */ + char* string=NULL; + int string_length; + + + /*recover inputs from pointers: */ + variables=*pvariables; + variables_descriptors=*pvariables_descriptors; + numvariables=*pnumvariables; + numresponses=*pnumresponses; + + /*numvariables: */ + MPI_Bcast(&numvariables,1,MPI_INT,0,MPI_COMM_WORLD); + + /*variables:*/ + if(my_rank!=0)variables=xNew(numvariables); + MPI_Bcast(variables,numvariables,MPI_DOUBLE,0,MPI_COMM_WORLD); + + /*variables_descriptors: */ + if(my_rank!=0){ + variables_descriptors=xNew(numvariables); + } + for(i=0;i(string_length); + MPI_Bcast(string,string_length,MPI_CHAR,0,MPI_COMM_WORLD); + if(my_rank!=0)variables_descriptors[i]=string; + } + + /*numresponses: */ + MPI_Bcast(&numresponses,1,MPI_INT,0,MPI_COMM_WORLD); + + /*Assign output pointers:*/ + *pnumvariables=numvariables; + *pvariables=variables; + *pvariables_descriptors=variables_descriptors; + *pnumresponses=numresponses; +} /*}}}*/ +void DakotaFree(double** pvariables,char*** pvariables_descriptors,char*** presponses_descriptors,int numvariables,int numresponses){ /*{{{*/ + + /*\brief DakotaFree: free allocations on other cpus, not done by Dakota.*/ + + int i; + extern int my_rank; + + double *variables = NULL; + char **variables_descriptors = NULL; + char **responses_descriptors = NULL; + char *string = NULL; + + /*recover pointers: */ + variables=*pvariables; + variables_descriptors=*pvariables_descriptors; + responses_descriptors=*presponses_descriptors; + + + /*Free variables and variables_descriptors only on cpu !=0*/ + if(my_rank!=0){ + xDelete(variables); + for(i=0;i(string); + } + xDelete(variables_descriptors); + } + + //responses descriptors on every cpu + for(i=0;i(string); + } + //rest of dynamic allocations. + xDelete(responses_descriptors); + + /*Assign output pointers:*/ + *pvariables=variables; + *pvariables_descriptors=variables_descriptors; + *presponses_descriptors=responses_descriptors; +} /*}}}*/ Index: /issm/trunk-jpl/src/c/solutions/dakota_core.cpp =================================================================== --- /issm/trunk-jpl/src/c/solutions/dakota_core.cpp (revision 13539) +++ /issm/trunk-jpl/src/c/solutions/dakota_core.cpp (revision 13539) @@ -0,0 +1,129 @@ +/*!\file: dakota_core.cpp + * \brief: wrapper to the Dakota capabilities. qmu fires up Dakota, and registers a Dakota Pluggin + * which will be in charge of running the solution sequences repeteadly, to garner statistics. + * + * This routine deals with running ISSM and Dakota in library mode. In library mode, Dakota does not + * run as an execuatble. Its capabilities are linked into the ISSM software. ISSM calls dakota routines + * directly from the dakota library. qmu.cpp is the code that is in charge of calling those routines. + * + * Dakota has its own way of running in parallel (for embarassingly parallel jobs). We do not want that, + * as ISSM knows exactly how to run "really parallel" jobs that use all CPUS. To bypass Dakota's parallelism, + * we overloaded the constructor for the parallel library (see the Dakota patch in the externalpackages/dakota + * directory). This overloaded constructor fires up Dakota serially on CPU 0 only! We take care of broadcasting + * to the other CPUS, hence ISSM is running in parallel, and Dakota serially on CPU0. + * + * Now, how does CPU 0 drive all other CPUS to carry out sensitivity analysese? By synchronizing its call to + * our ISSM cores (diagnostic_core, thermal_core, transient_core, etc ...) on CPU 0 with all other CPUS. + * This explains the structure of qmu.cpp, where cpu 0 runs Dakota, the Dakota pluggin fires up DakotaSpawnCore.cpp, + * while the other CPUS are waiting for a broadcast from CPU0, once they get it, they also fire up + * DakotaSpawnCore. In the end, DakotaSpawnCore is fired up on all CPUS, with CPU0 having Dakota inputs, that it will + * broacast to other CPUS. + * + * Now, how does dakota call the DakotaSpawnCore routine? The DakotaSpawnCore is embedded into the DakotaPlugin object + * which is derived from the Direct Interface Dakota objct. This is the only way to run Dakota in library + * mode (see their developper guide for more info). Dakota registers the DakotaPlugin object into its own + * database, and calls on the embedded DakotaSpawnCore from CPU0. + * + */ + +/*include files: {{{*/ +#ifdef HAVE_CONFIG_H +#include +#else +#error "Cannot compile with HAVE_CONFIG_H symbol! run configure first!" +#endif + +#include "./solutions.h" +#include "../shared/shared.h" +#include "../include/include.h" +#include "../io/io.h" +#include "../toolkits/toolkits.h" +#include "../EnumDefinitions/EnumDefinitions.h" +#include "../classes/dakota/DakotaPlugin.h" + +#ifdef _HAVE_DAKOTA_ //only works if dakota library has been compiled in. +#include "ParallelLibrary.H" +#include "ProblemDescDB.H" +#include "DakotaStrategy.H" +#include "DakotaModel.H" +#include "DakotaInterface.H" + +#endif +/*}}}*/ + +void dakota_core(FemModel* femmodel){ + + + #ifdef _HAVE_DAKOTA_ //only works if dakota library has been compiled in. + + extern int my_rank; + char* dakota_input_file = NULL; + char* dakota_output_file = NULL; + char* dakota_error_file = NULL; + int status=0; + Dakota::ModelLIter ml_iter; + Parameters* parameters = NULL; + + /*Retrieve parameters: */ + parameters=femmodel->parameters; + + /*Recover dakota_input_file, dakota_output_file and dakota_error_file, in the parameters dataset in parallel */ + parameters->FindParam(&dakota_input_file,QmuInNameEnum); + parameters->FindParam(&dakota_output_file,QmuOutNameEnum); + parameters->FindParam(&dakota_error_file,QmuErrNameEnum); + + if(my_rank==0){ + + // Instantiate/initialize the parallel library and problem description + // database objects. + Dakota::ParallelLibrary parallel_lib("serial"); //use our own ISSM Dakota library mode constructor, which only fires up Dakota on CPU 0. + Dakota::ProblemDescDB problem_db(parallel_lib); + + // Manage input file parsing, output redirection, and restart processing + // without a CommandLineHandler. This version relies on parsing of an + // input file. + problem_db.manage_inputs(dakota_input_file); + // specify_outputs_restart() is only necessary if specifying non-defaults + parallel_lib.specify_outputs_restart(dakota_output_file,dakota_error_file,NULL,NULL); + + // Instantiate the Strategy object (which instantiates all Model and + // Iterator objects) using the parsed information in problem_db. + Dakota::Strategy selected_strategy(problem_db); + + // convenience function for iterating over models and performing any + // interface plug-ins + Dakota::ModelList& models = problem_db.model_list(); + + for (ml_iter = models.begin(); ml_iter != models.end(); ml_iter++) { + + Dakota::Interface& interface = ml_iter->interface(); + + //set DB nodes to the existing Model specification + problem_db.set_db_model_nodes(ml_iter->model_id()); + + // Serial case: plug in derived Interface object without an analysisComm + interface.assign_rep(new SIM::DakotaPlugin(problem_db,(void*)femmodel), false); + } + + // Execute the strategy + problem_db.lock(); // prevent run-time DB queries + selected_strategy.run_strategy(); + + //Warn other cpus that we are done running the dakota iterator, by setting the counter to -1: + DakotaSpawnCore(NULL,0, NULL,NULL,0,femmodel,-1); + + } + else{ + + for(;;){ + if(!DakotaSpawnCore(NULL,0, NULL,NULL,0,femmodel,0))break; //counter came in at -1 on cpu0, bail out. + } + } + + /*Free ressources:*/ + xDelete(dakota_input_file); + xDelete(dakota_error_file); + xDelete(dakota_output_file); + + #endif //#ifdef _HAVE_DAKOTA_ +} Index: /issm/trunk-jpl/src/c/solutions/issm.cpp =================================================================== --- /issm/trunk-jpl/src/c/solutions/issm.cpp (revision 13538) +++ /issm/trunk-jpl/src/c/solutions/issm.cpp (revision 13539) @@ -7,4 +7,5 @@ void ProfilerEcho(Profiler* profiler); +void ProfilerEnd(Profiler* profiler, Results* results, Parameters* parameters); int main(int argc,char **argv){ @@ -38,7 +39,4 @@ /*profiling*/ Profiler* profiler=NULL; - bool profiling = false; - IssmPDouble Time_start, Flops_start; - IssmPDouble Solution_time, Memory_use, Current_flops; /*Initialize exception trapping: */ @@ -48,6 +46,7 @@ EnvironmentInit(argc,argv); - /*Some profiling: */ - profiler=new Profiler(); profiler->Tag(Start); + /*Start profiler: */ + profiler=new Profiler(); + profiler->Tag(Start); /*First process inputs*/ @@ -83,5 +82,4 @@ femmodel->parameters->FindParam(&control_analysis,InversionIscontrolEnum); femmodel->parameters->FindParam(&tao_analysis,InversionTaoEnum); - femmodel->parameters->FindParam(&profiling,DebugProfilingEnum); femmodel->parameters->FindParam(&autodiff,AutodiffIsautodiffEnum); @@ -106,11 +104,10 @@ _pprintLine_("call computational core:"); + + profiler->Tag(StartCore); - - if(profiling)ProfilingStart(&Time_start,&Flops_start); - if(dakota_analysis){ #ifdef _HAVE_DAKOTA_ - Dakotax(femmodel); + dakota_core(femmodel); #else _error_("ISSM was not compiled with dakota support, cannot carry out dakota analysis!"); @@ -119,21 +116,12 @@ else if(control_analysis){ #ifdef _HAVE_CONTROL_ - if(tao_analysis) - controltao_core(femmodel); - else - control_core(femmodel); + if(tao_analysis) controltao_core(femmodel); + else control_core(femmodel); #else - _error_("ISSM was not compiled with control support, cannot carry out dakota analysis!"); + _error_("ISSM was not compiled with control support, cannot carry out control analysis!"); #endif } else{ solutioncore(femmodel); - } - - if(profiling){ - ProfilingEnd(&Solution_time,&Memory_use,&Current_flops,Time_start,Flops_start); - femmodel->results->AddObject(new GenericExternalResult(femmodel->results->Size()+1, ProfilingSolutionTimeEnum, Solution_time, 1, 0)); - femmodel->results->AddObject(new GenericExternalResult(femmodel->results->Size()+1, ProfilingCurrentMemEnum, Memory_use, 1, 0)); - femmodel->results->AddObject(new GenericExternalResult(femmodel->results->Size()+1, ProfilingCurrentFlopsEnum, Current_flops, 1, 0)); } @@ -144,8 +132,8 @@ } #endif - profiler->Tag(FinishCore); - + ProfilerEnd(profiler,femmodel->results,femmodel->parameters); + _pprintLine_("write results to disk:"); OutputResultsx(femmodel->elements, femmodel->nodes, femmodel->vertices, femmodel->loads, femmodel->materials, femmodel->parameters,femmodel->results); @@ -183,6 +171,7 @@ delete femmodel; - /*Get finish time and close*/ - profiler->Tag(Finish); ProfilerEcho(profiler); + /*Profiling at the end: */ + profiler->Tag(Finish); + ProfilerEcho(profiler); /*Finalize environment:*/ @@ -196,19 +185,43 @@ } - - -void ProfilerEcho(Profiler* profiler){ - - _pprintLine_(""); - _pprintLine_(" "<Delta(StartInit,FinishInit)); - _pprintLine_(" "<Delta(StartCore,FinishCore)); +void ProfilerEcho(Profiler* profiler){ /*{{{*/ + + _pprintLine_(""); + _pprintLine_(" "<DeltaTime(StartInit,FinishInit)); + _pprintLine_(" "<DeltaTime(StartCore,FinishCore)); _pprintLine_(""); _pprintLine_(" Total elapsed time:" - <DeltaModHour(Start,Finish)<<" hrs " - <DeltaModMin(Start,Finish)<<" min " - <DeltaModSec(Start,Finish)<<" sec" + <DeltaTimeModHour(Start,Finish)<<" hrs " + <DeltaTimeModMin(Start,Finish)<<" min " + <DeltaTimeModSec(Start,Finish)<<" sec" ); _pprintLine_(""); -} - +} /*}}}*/ +void ProfilerEnd(Profiler* profiler, Results* results, Parameters* parameters){ /*{{{*/ + + bool profiling = false; + + IssmDouble solution_time; + IssmDouble solution_flops; + IssmDouble solution_memory; + + parameters->FindParam(&profiling,DebugProfilingEnum); + + if(profiling){ + + solution_time=profiler->DeltaTime(StartCore,FinishCore); + solution_flops=profiler->DeltaFlops(StartCore,FinishCore); + solution_memory=profiler->Memory(FinishCore); + + _pprintLine_("Solution elapsed time : " << solution_time << " Seconds"); + _pprintLine_("Solution elapsed flops : " << solution_flops << " Flops"); + _pprintLine_("Solution memory used : " << solution_memory << " Bytes"); + + /*Add to results: */ + results->AddObject(new GenericExternalResult(results->Size()+1, ProfilingSolutionTimeEnum, solution_time, 1, 0)); + results->AddObject(new GenericExternalResult(results->Size()+1, ProfilingCurrentMemEnum, solution_memory, 1, 0)); + results->AddObject(new GenericExternalResult(results->Size()+1, ProfilingCurrentFlopsEnum, solution_flops, 1, 0)); + } + +} /*}}}*/ Index: /issm/trunk-jpl/src/c/solutions/solutions.h =================================================================== --- /issm/trunk-jpl/src/c/solutions/solutions.h (revision 13538) +++ /issm/trunk-jpl/src/c/solutions/solutions.h (revision 13539) @@ -30,4 +30,5 @@ void steadystate_core(FemModel* femmodel); void transient_core(FemModel* femmodel); +void dakota_core(FemModel* femmodel); IssmDouble objectivefunction(IssmDouble search_scalar,OptArgs* optargs); @@ -47,4 +48,5 @@ void EnvironmentInit(int argc,char** argv); void EnvironmentFinalize(void); +int DakotaSpawnCore(double* responses, int numresponses, double* variables, char** variables_descriptors,int numvariables, void* femmodel,int counter); //solution configuration @@ -53,3 +55,4 @@ void AdjointCorePointerFromSolutionEnum(void (**padjointcore)(FemModel*),int solutiontype); + #endif