Index: /issm/trunk-jpl/src/c/main/issm_ocean.cpp =================================================================== --- /issm/trunk-jpl/src/c/main/issm_ocean.cpp (revision 21851) +++ /issm/trunk-jpl/src/c/main/issm_ocean.cpp (revision 21852) @@ -7,96 +7,44 @@ int main(int argc,char **argv){ -// /*diverse:*/ -// int nummodels = 2; -// int* commsizes=xNew(nummodels); -// int* rankzeros=xNew(nummodels); -// char** modelnames=xNew(nummodels); -// char** dirnames=xNew(nummodels); -// int iceid=0; -// int oceanid=1; + /*diverse:*/ + int icecommsize; + int rankzeros[2]; + int my_rank; + ISSM_MPI_Comm worldcomm; + // int modelid; -// int my_rank; // int count=0; -// ISSM_MPI_Comm worldcomm; -// ISSM_MPI_Comm modelcomm; -// ISSM_MPI_Comm toonceancomm; -// ISSM_MPI_Comm fromicecomms; + ISSM_MPI_Comm modelcomm; + ISSM_MPI_Comm frommitgcm; + ISSM_MPI_Comm tomitgcmcomm; /*Initialize exception trapping: */ ExceptionTrapBegin(); -// /*Initialize environment (MPI, PETSC, MUMPS, etc ...)*/ -// worldcomm=EnvironmentInit(argc,argv); -// -// /*What is my rank?:*/ -// ISSM_MPI_Comm_rank(worldcomm,&my_rank); -// -// /*First model is ice, second is ocean*/ -// for(int i=0;i(strlen(argv[5+3*i])+1); -// xMemCpy(string,argv[5+3*i],strlen(argv[5+3*i])+1); -// dirnames[i]=string; -// -// string=xNew(strlen(argv[5+3*i+1])+1); -// xMemCpy(string,argv[5+3*i+1],strlen(argv[5+3*i+1])+1); -// modelnames[i]=string; -// -// commsizes[i]=(int) strtol(argv[5+3*i+2], (char **)NULL, 10); -// } -// -// /*Figure out which model each cpu will belong to: */ -// count=0; -// for(int i=0;i=count && my_rank<(count+commsizes[i])){ -// modelid=i; -// break; -// } -// count+=commsizes[i]; -// } -// /*Buil array of who is rank 0 of their own group:*/ -// count=0; -// for(int i=0;i(4); -// arguments[0]=xNew(strlen(argv[0])+1); xMemCpy(arguments[0],argv[0],strlen(argv[0])+1); //executable name -// arguments[1]=xNew(strlen(argv[1])+1); xMemCpy(arguments[1],argv[1],strlen(argv[1])+1); //solution name -// arguments[2]=xNew(strlen(argv[5+3*modelid])+1); xMemCpy(arguments[2],argv[5+3*modelid],strlen(argv[5+3*modelid])+1); //directory name -// arguments[3]=xNew(strlen(argv[5+3*modelid+1])+1); xMemCpy(arguments[3],argv[5+3*modelid+1],strlen(argv[5+3*modelid+1])+1); //model name -// /*}}}*/ -// + /*First model is ice, second is ocean*/ + /*ice comm size: */ + icecommsize=(int) strtol(argv[2], (char **)NULL, 10); - //REMOVE - /*Initialize environment (MPI, PETSC, MUMPS, etc ...)*/ - ISSM_MPI_Comm comm_init=EnvironmentInit(argc,argv); + /*Buil array of who is rank 0 of their own group:*/ + rankzeros[0]=0; + rankzeros[1]=icecommsize; + + /*Split world into sub-communicators for each and every model:*/ + ISSM_MPI_Comm_split(worldcomm,0, my_rank, &modelcomm); + + ISSM_MPI_Intercomm_create( modelcomm, 0, worldcomm, rankzeros[1], 0, &tomitgcmcomm); + /*Initialize femmodel from arguments provided command line: */ - FemModel *femmodel = new FemModel(argc,argv,comm_init); - ///*Initialize femmodel from arguments provided command line: */ - //FemModel *femmodel = new FemModel(1,arguments,modelcomm); + FemModel *femmodel = new FemModel(argc,argv,modelcomm); -// /*Now that the models are initialized, keep communicator information in the parameters datasets of each model: */ -// femmodel->parameters->AddObject(new GenericParam(worldcomm,WorldCommEnum)); -// femmodel->parameters->AddObject(new IntParam(NumModelsEnum,nummodels)); -// femmodel->parameters->AddObject(new IntParam(ModelIdEnum,oceanid)); -// femmodel->parameters->AddObject(new IntParam(EarthIdEnum,iceid)); -// if(modelid==earthid) femmodel->parameters->AddObject(new GenericParam(fromicecomms,IcecapToEarthCommEnum)); -// else femmodel->parameters->AddObject(new GenericParam(toearthcomm,IcecapToEarthCommEnum)); + /*Now that the models are initialized, keep communicator information in the parameters datasets of each model: */ + femmodel->parameters->AddObject(new GenericParam(worldcomm,WorldCommEnum)); + femmodel->parameters->AddObject(new GenericParam(tomitgcmcomm,ToMITgcmCommEnum)); /*Solve: */ @@ -118,13 +66,6 @@ ExceptionTrapEnd(); -// /*Free ressources:*/ + /*Free ressources:*/ // xDelete(commsizes); -// for(int i=0;i(string); -// string=modelnames[i]; xDelete(string); -// } -// xDelete(dirnames); -// xDelete(modelnames); /*Return unix success: */ Index: /issm/trunk-jpl/src/c/shared/Enum/EnumDefinitions.h =================================================================== --- /issm/trunk-jpl/src/c/shared/Enum/EnumDefinitions.h (revision 21851) +++ /issm/trunk-jpl/src/c/shared/Enum/EnumDefinitions.h (revision 21852) @@ -16,4 +16,5 @@ WorldCommEnum, IcecapToEarthCommEnum, + ToMITgcmCommEnum, NumModelsEnum, ModelIdEnum, Index: /issm/trunk-jpl/src/c/shared/Enum/EnumToStringx.cpp =================================================================== --- /issm/trunk-jpl/src/c/shared/Enum/EnumToStringx.cpp (revision 21851) +++ /issm/trunk-jpl/src/c/shared/Enum/EnumToStringx.cpp (revision 21852) @@ -24,4 +24,5 @@ case WorldCommEnum : return "WorldComm"; case IcecapToEarthCommEnum : return "IcecapToEarthComm"; + case ToMITgcmCommEnum : return "ToMITgcmComm"; case NumModelsEnum : return "NumModels"; case ModelIdEnum : return "ModelId"; Index: /issm/trunk-jpl/src/c/shared/Enum/StringToEnumx.cpp =================================================================== --- /issm/trunk-jpl/src/c/shared/Enum/StringToEnumx.cpp (revision 21851) +++ /issm/trunk-jpl/src/c/shared/Enum/StringToEnumx.cpp (revision 21852) @@ -24,4 +24,5 @@ else if (strcmp(name,"WorldComm")==0) return WorldCommEnum; else if (strcmp(name,"IcecapToEarthComm")==0) return IcecapToEarthCommEnum; + else if (strcmp(name,"ToMITgcmComm")==0) return ToMITgcmCommEnum; else if (strcmp(name,"NumModels")==0) return NumModelsEnum; else if (strcmp(name,"ModelId")==0) return ModelIdEnum; @@ -136,9 +137,9 @@ else if (strcmp(name,"HydrologyshreveStabilization")==0) return HydrologyshreveStabilizationEnum; else if (strcmp(name,"Hydrologydc")==0) return HydrologydcEnum; - else if (strcmp(name,"SedimentHead")==0) return SedimentHeadEnum; else stage=2; } if(stage==2){ - if (strcmp(name,"SedimentHeadOld")==0) return SedimentHeadOldEnum; + if (strcmp(name,"SedimentHead")==0) return SedimentHeadEnum; + else if (strcmp(name,"SedimentHeadOld")==0) return SedimentHeadOldEnum; else if (strcmp(name,"SedimentHeadResidual")==0) return SedimentHeadResidualEnum; else if (strcmp(name,"EffectivePressure")==0) return EffectivePressureEnum; @@ -259,9 +260,9 @@ else if (strcmp(name,"NewDamage")==0) return NewDamageEnum; else if (strcmp(name,"StressIntensityFactor")==0) return StressIntensityFactorEnum; - else if (strcmp(name,"CalvingLaw")==0) return CalvingLawEnum; else stage=3; } if(stage==3){ - if (strcmp(name,"CalvingCalvingrate")==0) return CalvingCalvingrateEnum; + if (strcmp(name,"CalvingLaw")==0) return CalvingLawEnum; + else if (strcmp(name,"CalvingCalvingrate")==0) return CalvingCalvingrateEnum; else if (strcmp(name,"CalvingMeltingrate")==0) return CalvingMeltingrateEnum; else if (strcmp(name,"CalvingLevermann")==0) return CalvingLevermannEnum; @@ -382,9 +383,9 @@ else if (strcmp(name,"BalancethicknessOmega")==0) return BalancethicknessOmegaEnum; else if (strcmp(name,"BalancethicknessD0")==0) return BalancethicknessD0Enum; - else if (strcmp(name,"Smb")==0) return SmbEnum; else stage=4; } if(stage==4){ - if (strcmp(name,"SmbAnalysis")==0) return SmbAnalysisEnum; + if (strcmp(name,"Smb")==0) return SmbEnum; + else if (strcmp(name,"SmbAnalysis")==0) return SmbAnalysisEnum; else if (strcmp(name,"SmbSolution")==0) return SmbSolutionEnum; else if (strcmp(name,"SmbNumRequestedOutputs")==0) return SmbNumRequestedOutputsEnum; @@ -505,9 +506,9 @@ else if (strcmp(name,"MassFlux")==0) return MassFluxEnum; else if (strcmp(name,"MeltingOffset")==0) return MeltingOffsetEnum; - else if (strcmp(name,"Misfit")==0) return MisfitEnum; else stage=5; } if(stage==5){ - if (strcmp(name,"Pressure")==0) return PressureEnum; + if (strcmp(name,"Misfit")==0) return MisfitEnum; + else if (strcmp(name,"Pressure")==0) return PressureEnum; else if (strcmp(name,"PressurePicard")==0) return PressurePicardEnum; else if (strcmp(name,"AndroidFrictionCoefficient")==0) return AndroidFrictionCoefficientEnum; @@ -628,9 +629,9 @@ else if (strcmp(name,"Outputdefinition21")==0) return Outputdefinition21Enum; else if (strcmp(name,"Outputdefinition22")==0) return Outputdefinition22Enum; - else if (strcmp(name,"Outputdefinition23")==0) return Outputdefinition23Enum; else stage=6; } if(stage==6){ - if (strcmp(name,"Outputdefinition24")==0) return Outputdefinition24Enum; + if (strcmp(name,"Outputdefinition23")==0) return Outputdefinition23Enum; + else if (strcmp(name,"Outputdefinition24")==0) return Outputdefinition24Enum; else if (strcmp(name,"Outputdefinition25")==0) return Outputdefinition25Enum; else if (strcmp(name,"Outputdefinition26")==0) return Outputdefinition26Enum; @@ -751,10 +752,10 @@ else if (strcmp(name,"ToolkitsFileName")==0) return ToolkitsFileNameEnum; else if (strcmp(name,"RootPath")==0) return RootPathEnum; - else if (strcmp(name,"OutputFileName")==0) return OutputFileNameEnum; - else if (strcmp(name,"InputFileName")==0) return InputFileNameEnum; else stage=7; } if(stage==7){ - if (strcmp(name,"LockFileName")==0) return LockFileNameEnum; + if (strcmp(name,"OutputFileName")==0) return OutputFileNameEnum; + else if (strcmp(name,"InputFileName")==0) return InputFileNameEnum; + else if (strcmp(name,"LockFileName")==0) return LockFileNameEnum; else if (strcmp(name,"RestartFileName")==0) return RestartFileNameEnum; else if (strcmp(name,"ToolkitsOptionsAnalyses")==0) return ToolkitsOptionsAnalysesEnum; @@ -874,10 +875,10 @@ else if (strcmp(name,"Node")==0) return NodeEnum; else if (strcmp(name,"Numericalflux")==0) return NumericalfluxEnum; - else if (strcmp(name,"Neumannflux")==0) return NeumannfluxEnum; - else if (strcmp(name,"Param")==0) return ParamEnum; else stage=8; } if(stage==8){ - if (strcmp(name,"Moulin")==0) return MoulinEnum; + if (strcmp(name,"Neumannflux")==0) return NeumannfluxEnum; + else if (strcmp(name,"Param")==0) return ParamEnum; + else if (strcmp(name,"Moulin")==0) return MoulinEnum; else if (strcmp(name,"Pengrid")==0) return PengridEnum; else if (strcmp(name,"Penpair")==0) return PenpairEnum; @@ -997,10 +998,10 @@ else if (strcmp(name,"IceMass")==0) return IceMassEnum; else if (strcmp(name,"IceVolume")==0) return IceVolumeEnum; - else if (strcmp(name,"IceVolumeAboveFloatation")==0) return IceVolumeAboveFloatationEnum; - else if (strcmp(name,"TotalFloatingBmb")==0) return TotalFloatingBmbEnum; else stage=9; } if(stage==9){ - if (strcmp(name,"TotalGroundedBmb")==0) return TotalGroundedBmbEnum; + if (strcmp(name,"IceVolumeAboveFloatation")==0) return IceVolumeAboveFloatationEnum; + else if (strcmp(name,"TotalFloatingBmb")==0) return TotalFloatingBmbEnum; + else if (strcmp(name,"TotalGroundedBmb")==0) return TotalGroundedBmbEnum; else if (strcmp(name,"TotalSmb")==0) return TotalSmbEnum; else if (strcmp(name,"P0")==0) return P0Enum; Index: /issm/trunk-jpl/src/m/classes/clusters/localpfe.m =================================================================== --- /issm/trunk-jpl/src/m/classes/clusters/localpfe.m (revision 21851) +++ /issm/trunk-jpl/src/m/classes/clusters/localpfe.m (revision 21852) @@ -148,4 +148,22 @@ end %}}} + function BuildQueueScriptIceOcean(cluster,dirname,modelname,solution,io_gather,isvalgrind,isgprof,isdakota) % {{{ + + %write queuing script + %what is the executable being called? + executable='issm_ocean.exe'; + + fid=fopen([modelname '.queue'],'w'); + fprintf(fid,'#!%s\n',cluster.shell); + fprintf(fid,'mpiexec -np %i %s/%s %s %s %s: -np %i ./mitgcmuv\n',cluster.np,cluster.codepath,executable,solution,cluster.executionpath,modelname,cluster.npocean); + fclose(fid); + + %in interactive mode, create a run file, and errlog and outlog file + if cluster.interactive, + fid=fopen([modelname '.errlog'],'w'); fclose(fid); + fid=fopen([modelname '.outlog'],'w'); fclose(fid); + end + end + %}}} function BuildKrigingQueueScript(cluster,modelname,solution,io_gather,isvalgrind,isgprof) % {{{ @@ -226,4 +244,19 @@ end %}}} + function LaunchQueueJobIceOcean(cluster,modelname,dirname,filelist,restart,batch)% {{{ + + %figure out what shell extension we will use: + if isempty(strfind(cluster.shell,'csh')), + shellext='sh'; + else + shellext='csh'; + end + + if cluster.verbose, disp('launching solution sequence on remote cluster'); end + + launchcommand=['cd ' cluster.executionpath ' && rm -rf *.lock && tar -zxf ' dirname '.tar.gz && rm -rf *.tar.gz']; + issmssh(cluster.name,cluster.login,cluster.port,launchcommand); + + end %}}} function Download(cluster,dirname,filelist)% {{{ Index: /issm/trunk-jpl/src/m/solve/solveiceocean.m =================================================================== --- /issm/trunk-jpl/src/m/solve/solveiceocean.m (revision 21852) +++ /issm/trunk-jpl/src/m/solve/solveiceocean.m (revision 21852) @@ -0,0 +1,143 @@ +function md=solveiceocean(md,solutionstring,varargin) +%SOLVE - apply ice/ocean solution sequence for this model +% +% Usage: +% md=solveiceocean(md,solutionstring,varargin) +% where varargin is a lit of paired arguments of string OR enums +% +% solution types available comprise: +% - 'Stressbalance' or 'sb' +% - 'Masstransport' or 'mt' +% - 'Transient' or 'tr' +% +% extra options: +% - loadonly : does not solve. only load results +% - runtimename : true or false (default is true), makes name unique +% - checkconsistency : 'yes' or 'no' (default is 'yes'), ensures checks on consistency of model +% - restart: 'directory name (relative to the execution directory) where the restart file is located. +% - outbinread : if 0, download the outbin but do not process is (md.results is not updated) +% +% Examples: +% md=solve(md,'Transient'); +% md=solve(md,'tr'); + +if ~ischar(solutionstring) + error('ISSM''s solve function only accepts strings for solution sequences. Type help solve to get a list of supported solutions'); +end + +%recover and process solve options +if strcmpi(solutionstring,'sb') || strcmpi(solutionstring,'Stressbalance') + solutionstring = 'StressbalanceSolution'; +elseif strcmpi(solutionstring,'mt') || strcmpi(solutionstring,'Masstransport') + solutionstring = 'MasstransportSolution'; +elseif strcmpi(solutionstring,'tr') || strcmpi(solutionstring,'Transient') + solutionstring = 'TransientSolution'; +else + error(['solutionstring ' solutionstring ' not supported!']); +end +options=pairoptions(varargin{:},'solutionstring',solutionstring); + +%recover some fields +md.private.solution=solutionstring; +cluster=md.cluster; +if strcmpi(getfieldvalue(options,'batch','no'),'yes') batch=1; else batch=0; end +if ~isclass(cluster,'localpfe'), + error('cannot run ice/ocean simulation on any other cluster than localpfe'); +end + +%check model consistency +if strcmpi(getfieldvalue(options,'checkconsistency','yes'),'yes'), + if md.verbose.solution, + disp('checking model consistency'); + end + ismodelselfconsistent(md), +end + +%If we are restarting, actually use the provided runtime name: +restart=getfieldvalue(options,'restart',''); +%First, build a runtime name that is unique +if restart==1 + %Leave the runtimename as is +else + if ~isempty(restart), + md.private.runtimename=restart; + elseif getfieldvalue(options,'runtimename',true), + c=clock; + md.private.runtimename=sprintf('%s-%02i-%02i-%04i-%02i-%02i-%02i-%i',md.miscellaneous.name,c(2),c(3),c(1),c(4),c(5),floor(c(6)),feature('GetPid')); + else + md.private.runtimename=md.miscellaneous.name; + end +end + +%if running qmu analysis, some preprocessing of dakota files using models +%fields needs to be carried out. +if md.qmu.isdakota, + md=preqmu(md,options); +end + +%Do we load results only? +if getfieldvalue(options,'loadonly',false), + md=loadresultsfromcluster(md); + return; +end + +%Write all input files +marshall(md); % bin file +ToolkitsFile(md.toolkits,[md.miscellaneous.name '.toolkits']); % toolkits file +BuildQueueScriptIceOcean(cluster,md.private.runtimename,md.miscellaneous.name,md.private.solution,md.settings.io_gather,md.debug.valgrind,md.debug.gprof,md.qmu.isdakota); % queue file + +%Upload all required files +modelname = md.miscellaneous.name; +filelist = {[modelname '.bin '] [modelname '.toolkits ']}; +if ispc, + filelist{end+1}=[modelname '.bat ']; +else + filelist{end+1}=[modelname '.queue ']; +end + +if md.qmu.isdakota, + filelist{end+1} = [modelname '.qmu.in']; +end + +if isempty(restart), + UploadQueueJob(cluster,md.miscellaneous.name,md.private.runtimename,filelist); +end + +%launch queue job: +LaunchQueueJobIceOcean(cluster,md.miscellaneous.name,md.private.runtimename,filelist,restart,batch); + +%return if batch: +if batch, + if md.verbose.solution, + disp('batch mode requested: not launching job interactively'); + disp('launch solution sequence on remote cluster by hand'); + end + return; +end +%wait on lock +if isnan(md.settings.waitonlock), + %load when user enters 'y' + disp('solution launched on remote cluster. log in to detect job completion.'); + choice=input('Is the job successfully completed? (y/n)','s'); + if ~strcmp(choice,'y'), + disp('Results not loaded... exiting'); + else + md=loadresultsfromcluster(md); + end +elseif md.settings.waitonlock>0, + %we wait for the done file + done=waitonlock(md); + if md.verbose.solution, + disp('loading results from cluster'); + end + md=loadresultsfromcluster(md); +elseif md.settings.waitonlock==0, + disp('Model results must be loaded manually with md=loadresultsfromcluster(md);'); +end + +%post processes qmu results if necessary +if md.qmu.isdakota, + if ~strncmpi(getfieldvalue(options,'keep','y'),'y',1) + system(['rm -rf qmu' num2str(feature('GetPid'))]); + end +end