/*!\file:  dakota_core.cpp
 * \brief: wrapper to the Dakota capabilities. qmu fires up Dakota, and registers a Dakota Pluggin
 * which will be in charge of running the solution sequences repeteadly, to garner statistics. 
 *
 * This routine deals with running ISSM and Dakota in library mode. In library mode, Dakota does not 
 * run as an execuatble. Its capabilities are linked into the ISSM software. ISSM calls dakota routines 
 * directly from the dakota library. qmu.cpp is the code that is in charge of calling those routines. 
 *
 * Dakota has its own way of running in parallel (for embarassingly parallel jobs). We do not want that, 
 * as ISSM knows exactly how to run "really parallel" jobs that use all CPUS. To bypass Dakota's parallelism, 
 * we overloaded the constructor for the parallel library (see the Dakota patch in the externalpackages/dakota
 * directory). This overloaded constructor fires up Dakota serially on CPU 0 only! We take care of broadcasting 
 * to the other CPUS, hence ISSM is running in parallel, and Dakota serially on CPU0. 
 *
 * Now, how does CPU 0 drive all other CPUS to carry out sensitivity analysese? By synchronizing its call to 
 * our ISSM cores (stressbalance_core, thermal_core, transient_core, etc ...) on CPU 0 with all other CPUS. 
 * This explains the structure of qmu.cpp, where cpu 0 runs Dakota, the Dakota pluggin fires up DakotaSpawnCore.cpp, 
 * while the other CPUS are waiting for a broadcast from CPU0, once they get it, they also fire up 
 * DakotaSpawnCore. In the end, DakotaSpawnCore is fired up on all CPUS, with CPU0 having Dakota inputs, that it will 
 * broacast to other CPUS. 
 *
 * Now, how does dakota call the DakotaSpawnCore routine? The DakotaSpawnCore is embedded into the DakotaPlugin object 
 * which is derived from the Direct Interface Dakota objct. This is the only way to run Dakota in library 
 * mode (see their developper guide for more info). Dakota registers the DakotaPlugin object into its own 
 * database, and calls on the embedded DakotaSpawnCore from CPU0. 
 *
 */ 

/*include files: {{{*/
#ifdef HAVE_CONFIG_H
#include <config.h>
#else
#error "Cannot compile with HAVE_CONFIG_H symbol! run configure first!"
#endif

#include "./cores.h"
#include "../toolkits/toolkits.h"
#include "../shared/shared.h"
#include "../classes/classes.h"

#ifdef _HAVE_DAKOTA_ //only works if dakota library has been compiled in.
#if _DAKOTA_MAJOR_ < 5 || (_DAKOTA_MAJOR_ == 5 && _DAKOTA_MINOR_ < 3)
#include <ParallelLibrary.H>
#include <ProblemDescDB.H>
#include <DakotaStrategy.H>
#include <DakotaModel.H>
#include <DakotaInterface.H>
#else
#include <ParallelLibrary.hpp>
#include <ProblemDescDB.hpp>
#include <DakotaStrategy.hpp>
#include <DakotaModel.hpp>
#include <DakotaInterface.hpp>
#endif
#include "./DakotaSpawnCore.h"
#endif
/*}}}*/
/*DakotaPlugin class {{{*/
#ifdef _HAVE_DAKOTA_ //only works if dakota library has been compiled in.
#if _DAKOTA_MAJOR_ < 5 || (_DAKOTA_MAJOR_ == 5 && _DAKOTA_MINOR_ < 3)
#include <DirectApplicInterface.H>
#include <DakotaResponse.H>
#include <ParamResponsePair.H>
#include <system_defs.h>
#include <ProblemDescDB.H>
#include <ParallelLibrary.H>
#else
#include <DirectApplicInterface.hpp>
#include <DakotaResponse.hpp>
#include <ParamResponsePair.hpp>
#include <ProblemDescDB.hpp>
#include <ParallelLibrary.hpp>
#endif
namespace SIM {
	class DakotaPlugin: public Dakota::DirectApplicInterface{
		public:
			/*these fields are used by core solutions: */
			void *femmodel;
			int   counter;
			/*Constructors/Destructors*/
			DakotaPlugin(const Dakota::ProblemDescDB& problem_db,void* in_femmodel):Dakota::DirectApplicInterface(problem_db){/*{{{*/
				femmodel = in_femmodel;
				counter  = 0;
			}/*}}}*/
			~DakotaPlugin(){/*{{{*/
				/* Virtual destructor handles referenceCount at Interface level. */ 
			}/*}}}*/
		protected:
			/*execute the input filter portion of a direct evaluation invocation*/
			//int derived_map_if(const Dakota::String& if_name);
			/*execute an analysis code portion of a direct evaluation invocation*/
			int derived_map_ac(const Dakota::String& driver){/*{{{*/

				int i;
				IssmDouble* variables=NULL;
				char** variable_descriptors=NULL;
				char*  variable_descriptor=NULL;
				IssmDouble* responses=NULL;

				/*increae counter: */
				counter++;

				/*Before launching analysis, we need to transfer the dakota inputs into Issm 
				 *readable variables: */

				/*First, the variables: */
				variables=xNew<IssmDouble>(numACV);
				for(i=0;i<numACV;i++){
					variables[i]=xC[i];
				}
				/*The descriptors: */
				variable_descriptors=xNew<char*>(numACV);
				for(i=0;i<numACV;i++){
					std::string label=xCLabels[i];
					variable_descriptor=xNew<char>(strlen(label.c_str())+1);
					memcpy(variable_descriptor,label.c_str(),(strlen(label.c_str())+1)*sizeof(char));

					variable_descriptors[i]=variable_descriptor;
				}

				/*Initialize responses: */
				responses=xNewZeroInit<IssmDouble>(numFns);

				/*run core solution: */
				DakotaSpawnCore(responses,numFns, variables,variable_descriptors,numACV,femmodel,counter);

				/*populate responses: */
				for(i=0;i<numFns;i++){
					fnVals[i]=responses[i];
				}

				/*Free ressources:*/
				xDelete<IssmDouble>(variables);
				for(i=0;i<numACV;i++){
					variable_descriptor=variable_descriptors[i];
					xDelete<char>(variable_descriptor);
				}
				xDelete<char*>(variable_descriptors);
				xDelete<IssmDouble>(responses);

				return 0;
			}/*}}}*/
			/*execute the output filter portion of a direct evaluation invocation*/
			//int derived_map_of(const Dakota::String& of_name);
			/*add for issm: */
			int GetCounter(){/*{{{*/
				return counter;
			}/*}}}*/
		private:
	};
} 
#endif
/*}}}*/

void dakota_core(FemModel* femmodel){ 

	#ifdef _HAVE_DAKOTA_ //only works if dakota library has been compiled in.

	int                my_rank;
	char              *dakota_input_file  = NULL;
	char              *dakota_output_file = NULL;
	char              *dakota_error_file  = NULL;
	Dakota::ModelLIter ml_iter;

	/*Recover dakota_input_file, dakota_output_file and dakota_error_file, in the parameters dataset in parallel */
	femmodel->parameters->FindParam(&dakota_input_file,QmuInNameEnum);
	femmodel->parameters->FindParam(&dakota_output_file,QmuOutNameEnum);
	femmodel->parameters->FindParam(&dakota_error_file,QmuErrNameEnum);

	/*recover my_rank:*/
	my_rank=IssmComm::GetRank();

	if(my_rank==0){

		// Instantiate/initialize the parallel library and problem description
		// database objects.
		char* dakotamode=xNew<char>(strlen("serial")+1);
		xMemCpy<char>(dakotamode,"serial",strlen("serial")+1);
		Dakota::ParallelLibrary parallel_lib(dakotamode); //use our own ISSM Dakota library mode constructor, which only fires up Dakota on CPU 0. 
		Dakota::ProblemDescDB problem_db(parallel_lib); 
		xDelete<char>(dakotamode);

		// Manage input file parsing, output redirection, and restart processing
		// without a CommandLineHandler.  This version relies on parsing of an
		// input file.
		problem_db.manage_inputs(dakota_input_file);
		// specify_outputs_restart() is only necessary if specifying non-defaults
		parallel_lib.specify_outputs_restart(dakota_output_file,dakota_error_file,NULL,NULL);

		// Instantiate the Strategy object (which instantiates all Model and
		// Iterator objects) using the parsed information in problem_db.
		Dakota::Strategy selected_strategy(problem_db);

		// convenience function for iterating over models and performing any
		// interface plug-ins
		Dakota::ModelList& models = problem_db.model_list();

		for (ml_iter = models.begin(); ml_iter != models.end(); ml_iter++) {

			Dakota::Interface& interface = ml_iter->interface();

			//set DB nodes to the existing Model specification
			problem_db.set_db_model_nodes(ml_iter->model_id());

			// Serial case: plug in derived Interface object without an analysisComm
			interface.assign_rep(new SIM::DakotaPlugin(problem_db,(void*)femmodel), false);
		}

		// Execute the strategy
		problem_db.lock(); // prevent run-time DB queries
		selected_strategy.run_strategy();

		//Warn other cpus that we are done running the dakota iterator, by setting the counter to -1:
		DakotaSpawnCore(NULL,0, NULL,NULL,0,femmodel,-1);

	}
	else{

		for(;;){
			if(!DakotaSpawnCore(NULL,0, NULL,NULL,0,femmodel,0))break; //counter came in at -1 on cpu0, bail out.
		}
	}

	/*Free ressources:*/
	xDelete<char>(dakota_input_file);
	xDelete<char>(dakota_error_file);
	xDelete<char>(dakota_output_file);

	#endif //#ifdef _HAVE_DAKOTA_
}
