/*!\file:  dakota_core.cpp
 * \brief: wrapper to the Dakota capabilities. qmu fires up Dakota, and registers a Dakota Pluggin
 * which will be in charge of running the solution sequences repeteadly, to garner statistics. 
 *
 * This routine deals with running ISSM and Dakota in library mode. In library mode, Dakota does not 
 * run as an execuatble. Its capabilities are linked into the ISSM software. ISSM calls dakota routines 
 * directly from the dakota library. qmu.cpp is the code that is in charge of calling those routines. 
 *
 * Dakota has its own way of running in parallel (for embarassingly parallel jobs). We do not want that, 
 * as ISSM knows exactly how to run "really parallel" jobs that use all CPUS. To bypass Dakota's parallelism, 
 * we overloaded the constructor for the parallel library (see the Dakota patch in the externalpackages/dakota
 * directory). This overloaded constructor fires up Dakota serially on CPU 0 only! We take care of broadcasting 
 * to the other CPUS, hence ISSM is running in parallel, and Dakota serially on CPU0. 
 *
 * Now, how does CPU 0 drive all other CPUS to carry out sensitivity analysese? By synchronizing its call to 
 * our ISSM cores (diagnostic_core, thermal_core, transient_core, etc ...) on CPU 0 with all other CPUS. 
 * This explains the structure of qmu.cpp, where cpu 0 runs Dakota, the Dakota pluggin fires up DakotaSpawnCore.cpp, 
 * while the other CPUS are waiting for a broadcast from CPU0, once they get it, they also fire up 
 * DakotaSpawnCore. In the end, DakotaSpawnCore is fired up on all CPUS, with CPU0 having Dakota inputs, that it will 
 * broacast to other CPUS. 
 *
 * Now, how does dakota call the DakotaSpawnCore routine? The DakotaSpawnCore is embedded into the DakotaPlugin object 
 * which is derived from the Direct Interface Dakota objct. This is the only way to run Dakota in library 
 * mode (see their developper guide for more info). Dakota registers the DakotaPlugin object into its own 
 * database, and calls on the embedded DakotaSpawnCore from CPU0. 
 *
 */ 

/*include files: {{{*/
#ifdef HAVE_CONFIG_H
#include <config.h>
#else
#error "Cannot compile with HAVE_CONFIG_H symbol! run configure first!"
#endif

#include "./solutions.h"
#include "../shared/shared.h"
#include "../include/include.h"
#include "../io/io.h"
#include "../toolkits/toolkits.h"
#include "../EnumDefinitions/EnumDefinitions.h"
#include "../classes/dakota/DakotaPlugin.h"

#ifdef _HAVE_DAKOTA_ //only works if dakota library has been compiled in.
#include <ParallelLibrary.H>
#include <ProblemDescDB.H>
#include <DakotaStrategy.H>
#include <DakotaModel.H>
#include <DakotaInterface.H>
#endif
/*}}}*/

void dakota_core(FemModel* femmodel){ 

	#ifdef _HAVE_DAKOTA_ //only works if dakota library has been compiled in.

	int                my_rank;
	char*              dakota_input_file  = NULL;
	char*              dakota_output_file = NULL;
	char*              dakota_error_file  = NULL;
	int                status=0;
	Dakota::ModelLIter ml_iter;
	Parameters* parameters                = NULL;

	/*Recover dakota_input_file, dakota_output_file and dakota_error_file, in the parameters dataset in parallel */
	femmodel->parameters->FindParam(&dakota_input_file,QmuInNameEnum);
	femmodel->parameters->FindParam(&dakota_output_file,QmuOutNameEnum);
	femmodel->parameters->FindParam(&dakota_error_file,QmuErrNameEnum);

	/*recover my_rank:*/
	my_rank=IssmComm::GetRank();

	if(my_rank==0){

		// Instantiate/initialize the parallel library and problem description
		// database objects.
		Dakota::ParallelLibrary parallel_lib("serial"); //use our own ISSM Dakota library mode constructor, which only fires up Dakota on CPU 0. 
		Dakota::ProblemDescDB problem_db(parallel_lib); 

		// Manage input file parsing, output redirection, and restart processing
		// without a CommandLineHandler.  This version relies on parsing of an
		// input file.
		problem_db.manage_inputs(dakota_input_file);
		// specify_outputs_restart() is only necessary if specifying non-defaults
		parallel_lib.specify_outputs_restart(dakota_output_file,dakota_error_file,NULL,NULL);

		// Instantiate the Strategy object (which instantiates all Model and
		// Iterator objects) using the parsed information in problem_db.
		Dakota::Strategy selected_strategy(problem_db);

		// convenience function for iterating over models and performing any
		// interface plug-ins
		Dakota::ModelList& models = problem_db.model_list();

		for (ml_iter = models.begin(); ml_iter != models.end(); ml_iter++) {

			Dakota::Interface& interface = ml_iter->interface();

			//set DB nodes to the existing Model specification
			problem_db.set_db_model_nodes(ml_iter->model_id());

			// Serial case: plug in derived Interface object without an analysisComm
			interface.assign_rep(new SIM::DakotaPlugin(problem_db,(void*)femmodel), false);
		}

		// Execute the strategy
		problem_db.lock(); // prevent run-time DB queries
		selected_strategy.run_strategy();

		//Warn other cpus that we are done running the dakota iterator, by setting the counter to -1:
		DakotaSpawnCore(NULL,0, NULL,NULL,0,femmodel,-1);

	}
	else{

		for(;;){
			if(!DakotaSpawnCore(NULL,0, NULL,NULL,0,femmodel,0))break; //counter came in at -1 on cpu0, bail out.
		}
	}

	/*Free ressources:*/
	xDelete<char>(dakota_input_file);
	xDelete<char>(dakota_error_file);
	xDelete<char>(dakota_output_file);

	#endif //#ifdef _HAVE_DAKOTA_
}
