Index: /issm/trunk/src/m/clusters/QueueRequirements.m =================================================================== --- /issm/trunk/src/m/clusters/QueueRequirements.m (revision 5954) +++ /issm/trunk/src/m/clusters/QueueRequirements.m (revision 5954) @@ -0,0 +1,46 @@ +function QueueRequirements(available_queues,queue_requirements_time,queue_requirements_np,queue,np,time) +%QUEUEREQUIREMENTS queue requirements in time, number of cpus, by name of queue. +% +% +% Usage: QueueRequirements(available_queues,queue_requirements_time,queue_requirements_np,np,time) +% + + +%Ok, go through requirements for current queue: +index=ismemberi(queue,available_queues); +if ~index, + %ok, either we a generic cluster, with 'none' queue, or we could not find the queue reqruirements + if strcmpi(available_queues{1},'none'), + %reset index to 1, so we can fish the requirements + index=1; + else + string=available_queues{1}; + for i=2:length(available_queues), + string=[string ' ' available_queues{i}]; + end + error(['QueueRequirements error message: availables queues are ' string]); + end +end + +%check on time requirements +rtime=queue_requirements_time(index); + +if time<=0, + error('QueueRequirements: time should be a positive number'); +end + +if time>rtime, + error(['QueueRequirements: time should be < ' num2str(rtime) ' for queue: ' queue]); +end + +%check on np requirements +rnp=queue_requirements_np(index); +rmod=queue_requirements_modulo(index); + +if np<=0, + error('QueueRequirements: np should be a positive number'); +end + +if np>rnp, + error(['QueueRequirements: np should be < ' num2str(rnp) ' for queue: ' queue]); +end Index: /issm/trunk/src/m/clusters/README =================================================================== --- /issm/trunk/src/m/clusters/README (revision 5954) +++ /issm/trunk/src/m/clusters/README (revision 5954) @@ -0,0 +1,13 @@ +This directory holds classes corresponding to every cluster +that can be used. + +Why classes? +Because handling the specificity of every cluster is hard in a general +code. Having classes with generic methods, such as: BuildQueueScript, +LaunchQueueJob, and internal data corresponding to every cluster's specificity +is much more manageable and powerful. + + +How to add your cluster? +use an existing cluster script, such as pleiades.m, and rename it to your cluster name. +update the methods. Index: /issm/trunk/src/m/clusters/astrid.m =================================================================== --- /issm/trunk/src/m/clusters/astrid.m (revision 5954) +++ /issm/trunk/src/m/clusters/astrid.m (revision 5954) @@ -0,0 +1,66 @@ +%ASTRID class definition +% +% Usage: +% cluster=astrid(); +% +classdef astrid + properties (SetAccess=public) + %{{{1 + name='astrid' + np =15; %number of processors + codepath=[issmdir '/bin']; + executionpath=[issmdir '../execution']; + valgrind=[issmdir '/externalpackages/valgrind/install/bin/valgrind']; + valgrindlib=[issmdir '/externalpackages/valgrind/install/lib/libmpidebug.so']; + valgrindsup=[issmdir 'externalpackages/valgrind/issm.supp']; + login='larour'; + port=0; + %}}} + end + methods + function []=disp(md) %{{{1 + % display the object + disp(sprintf('class ''%s'' object ''%s'' = ',class(md),inputname(1))); + disp(sprintf(' name: %s',md.name)); + disp(sprintf(' np: %i',md.np)); + disp(sprintf(' codepath: %s',md.codepath)); + disp(sprintf(' executionpath: %s',md.executionpath)); + disp(sprintf(' valgrind: %s',md.valgrind)); + disp(sprintf(' valgrindlib: %s',md.valgrindlib)); + disp(sprintf(' valgrindsup: %s',md.valgrindsup)); + end + %}}} + function IsConsistent(md) %{{{1 + if md.np>16, + error('IsConsistent error message: number of processors should be lest than 16!'); + end + if isnan(md.np), + error('IsConsistent error message: number of processors should not be NaN!'); + end + end + %}}} + function BuildQueueScript(md,modelname,analysis_type,mem_debug) %{{{1 + + %open file for writing: + fid=fopen([modelname '.queue'],'w'); + + %write instructions for launching a job on the cluster + fprintf(fid,'#!/bin/sh\n'); + if mem_debug==0, + fprintf(fid,'mpirun -np %i %s/issm.exe %s %s %s.bin %s.outbin %s.lock 2> %s.errlog >%s.outlog & ',md.np,md.codepath,EnumToString(analysis_type),md.executionpath,modelname,modelname,modelname,modelname,modelname); + else + %fprintf(fid,'LD_PRELOAD=%s mpirun -np %i %s --leak-check=full --gen-suppressions=all --suppressions=%s %s/issm.exe %s %s %s.bin %s.outbin %s.lock 2> %s.errlog >%s.outlog & ',md.valgrindlib,md.np,md.valgrind,md.valgrindsup,md.codepath,EnumToString(analysis_type),md.executionpath,modelname,modelname,modelname,modelname,modelname); + fprintf(fid,'LD_PRELOAD=%s mpirun -np %i %s --leak-check=full --suppressions=%s %s/issm.exe %s %s %s.bin %s.outbin %s.lock 2> %s.errlog >%s.outlog & ',md.valgrindlib,md.np,md.valgrind,md.valgrindsup, md.codepath,EnumToString(analysis_type),md.executionpath,modelname,modelname,modelname,modelname,modelname); + end + + %close file + fclose(fid); + + end + %}}} + function command=LaunchCommand(md,modelruntimename,modelname)%{{{1 + command=['cd ' md.executionpath ' && rm -rf ./' modelruntimename ' && mkdir ' modelruntimename ' && cd ' modelruntimename ' && mv ../' modelruntimename '.tar.gz ./ && tar -zxf ' modelruntimename '.tar.gz && source ' modelname '.queue ']; + end + %}}} + end +end Index: /issm/trunk/src/m/clusters/castor.m =================================================================== --- /issm/trunk/src/m/clusters/castor.m (revision 5954) +++ /issm/trunk/src/m/clusters/castor.m (revision 5954) @@ -0,0 +1,73 @@ +%CASTOR class definition +% +% Usage: +% cluster=castor(); +% +classdef castor + properties (SetAccess=public) + %{{{1 + name='castor' + np =128; %number of processors + queue='shortc'; + time=180; + codepath=/workp/edw/larour/issm-2.0/bin + executionpath=/workp/edw/larour/Testing/Execution + login='larour'; + port=0; + %}}} + end + methods + function []=disp(md) %{{{1 + % display the object + disp(sprintf('class ''%s'' object ''%s'' = ',class(md),inputname(1))); + disp(sprintf(' name: %s',md.name)); + disp(sprintf(' np: %i',md.np)); + disp(sprintf(' queue: %s',md.queue)); + disp(sprintf(' time: %i',md.time)); + disp(sprintf(' codepath: %s',md.codepath)); + disp(sprintf(' executionpath: %s',md.executionpath)); + end + %}}} + function IsConsistent(md) %{{{1 + + available_queues={'shortc','longc'}; + queue_requirements_time=[180 720]; + queue_requirements_np=[128 128]; + + QueueRequirements(available_queues,queue_requirements_time,queue_requirements_np,md.queue,md.np,md.time) + end + %}}} + function BuildQueueScript(md,modelname,analysis_type,mem_debug) %{{{1 + + %open file for writing: + fid=fopen([modelname '.queue'],'w'); + + fprintf(fid,'#!/bin/sh\n'); + fprintf(fid,'#PBS -l walltime=%i\n',md.time*60); %walltime is in seconds. + fprintf(fid,'#PBS -N %s\n',modelname); + fprintf(fid,'#PBS -l ncpus=%i\n',md.np); + if ~isempty(queue), + fprintf(fid,'#PBS -q %s\n',md.queue); + end + fprintf(fid,'#PBS -o %s.outlog \n',modelname); + fprintf(fid,'#PBS -e %s.errlog \n',modelname); + + fprintf(fid,'export PBS_O_WORKDIR=%s\n',md.executionpath); + fprintf(fid,'cd $PBS_O_WORKDIR\n'); + fprintf(fid,'export OMP_NUM_THREADS=1\n'); + fprintf(fid,'dplace -s1 -c0-%i mpirun -np %i %s/issm.exe %s %s %s.bin %s.outbin %s.lock',md.np-1,md.np,md.codepath,EnumToString(analysis_type),md.executionpath,modelname,modelname,modelname); + + %close file + fclose(fid); + + end + %}}} + function command=LaunchCommand(md,modelruntimename,modelname)%{{{1 + command=['cd ' md.executionpath ' && rm -rf ./' modelruntimename ' && mkdir ' modelruntimename ' && cd ' modelruntimename ' && mv ../' modelruntimename '.tar.gz ./ && tar -zxf ' modelruntimename '.tar.gz && qsub ' modelname '.queue ']; + end + %}}} + end +end + + + Index: /issm/trunk/src/m/clusters/cluster.rc =================================================================== --- /issm/trunk/src/m/clusters/cluster.rc (revision 5954) +++ /issm/trunk/src/m/clusters/cluster.rc (revision 5954) @@ -0,0 +1,28 @@ +begin + +#This file sets default platform settings used by ISSM. +#This file should always start with begin, and end with end. +#Comments start with #. + +#The syntax to add a computational cluster is the following: +#cluster_name=foobar +#cluster_codepath=/home/foobar/issm/bin +#cluster_executionpath=/home/foobar/issm/bin + +#Just uncomment the 3 lines above if you want to add a cluster +#with hostname foobar, where issm code has been installed in +#cluster_codepath, and where execution will be carried out in cluster_executionpath +#The cluster_name can be of any form, provided it's a string. Using +#the network hostname is good practice. + +#Default + +#Final remark on including cluster settings: do not include spaces +#between a field, the '=' sign and its value. Do not also include trailing spaces +#at the end of a line. + +cluster_name=pfe +cluster_codepath=/staff/elarour/trunk/bin +cluster_executionpath=/nobackupp10/elarour/Testing +cluster_login=elarour +cluster_port=1025 Index: /issm/trunk/src/m/clusters/cosmos.m =================================================================== --- /issm/trunk/src/m/clusters/cosmos.m (revision 5954) +++ /issm/trunk/src/m/clusters/cosmos.m (revision 5954) @@ -0,0 +1,69 @@ +%COSMOS class definition +% +% Usage: +% cluster=cosmos(); +% +classdef cosmos + properties (SetAccess=public) + %{{{1 + name='cosmos' + np =128; %number of processors + queue='shortq'; + time=3*60; + codepath='/work00/edw/larour/issm-2.0/bin'; + executionpath='/work00/edw/larour/Execution'; + login='larour'; + port=0; + %}}} + end + methods + function []=disp(md) %{{{1 + % display the object + disp(sprintf('class ''%s'' object ''%s'' = ',class(md),inputname(1))); + disp(sprintf(' name: %s',md.name)); + disp(sprintf(' np: %i',md.np)); + disp(sprintf(' queue: %s',md.queue)); + disp(sprintf(' time: %i',md.time)); + disp(sprintf(' codepath: %s',md.codepath)); + disp(sprintf(' executionpath: %s',md.executionpath)); + end + %}}} + function IsConsistent(md) %{{{1 + + available_queues={'debug','shortq','longq'}; + queue_requirements_time=[60*1 60*3 60*17]; + queue_requirements_np=[32 128 256]; + + QueueRequirements(available_queues,queue_requirements_time,queue_requirements_np,md.queue,md.np,md.time) + end + %}}} + function BuildQueueScript(md,modelname,analysis_type,mem_debug) %{{{1 + + %open file for writing: + fid=fopen([modelname '.queue'],'w'); + + fprintf(fid,'#!/bin/bash\n'); + fprintf(fid,'#PBS -l select=%i:ncpus=1\n',md.np); + fprintf(fid,'#PBS -N %s\n',modelname); + fprintf(fid,'#PBS -l walltime=%i\n',time*60); %walltime is in seconds. + fprintf(fid,'#PBS -q %s\n',queue); + fprintf(fid,'#PBS -o %s.outlog \n',modelname); + fprintf(fid,'#PBS -e %s.errlog \n',modelname); + fprintf(fid,'export PBS_O_WORKDIR=%s\n',md.executionpath); + fprintf(fid,'cd $PBS_O_WORKDIR\n'); + fprintf(fid,'export OMP_NUM_THREADS=1\n'); + fprintf(fid,'ulimit -s unlimited\n'); + fprintf(fid,'ulimit -c 0\n'); + fprintf(fid,'/opt/mpich/gm/intel10.1/bin/mpirun -np %i %s/issm.exe %s %s %s.bin %s.outbin %s.lock',md.np,md.codepath,EnumToString(analysis_type),md.executionpath,modelname,modelname,modelname); + + %close file + fclose(fid); + + end + %}}} + function command=LaunchCommand(md,modelruntimename,modelname)%{{{1 + command=['cd ' md.executionpath ' && rm -rf ./' modelruntimename ' && mkdir ' modelruntimename ' && cd ' modelruntimename ' && mv ../' modelruntimename '.tar.gz ./ && tar -zxf ' modelruntimename '.tar.gz && qsub -S/bin/sh ' modelname '.queue ']; + end + %}}} + end +end Index: /issm/trunk/src/m/clusters/eric-mac.m =================================================================== --- /issm/trunk/src/m/clusters/eric-mac.m (revision 5954) +++ /issm/trunk/src/m/clusters/eric-mac.m (revision 5954) @@ -0,0 +1,66 @@ +%ERIC-MAC class definition +% +% Usage: +% cluster=eric-mac(); +% +classdef astrid + properties (SetAccess=public) + %{{{1 + name='eric-mac' + np =3; %number of processors + codepath=[issmdir '/bin']; + executionpath=[issmdir '../execution']; + valgrind=[issmdir '/externalpackages/valgrind/install/bin/valgrind']; + valgrindlib=[issmdir '/externalpackages/valgrind/install/lib/libmpidebug.so']; + valgrindsup=[issmdir 'externalpackages/valgrind/issm.supp']; + login='larour'; + port=0; + %}}} + end + methods + function []=disp(md) %{{{1 + % display the object + disp(sprintf('class ''%s'' object ''%s'' = ',class(md),inputname(1))); + disp(sprintf(' name: %s',md.name)); + disp(sprintf(' np: %i',md.np)); + disp(sprintf(' codepath: %s',md.codepath)); + disp(sprintf(' executionpath: %s',md.executionpath)); + disp(sprintf(' valgrind: %s',md.valgrind)); + disp(sprintf(' valgrindlib: %s',md.valgrindlib)); + disp(sprintf(' valgrindsup: %s',md.valgrindsup)); + end + %}}} + function IsConsistent(md) %{{{1 + if md.np>4, + error('IsConsistent error message: number of processors should be lest than 16!'); + end + if isnan(md.np), + error('IsConsistent error message: number of processors should not be NaN!'); + end + end + %}}} + function BuildQueueScript(md,modelname,analysis_type,mem_debug) %{{{1 + + %open file for writing: + fid=fopen([modelname '.queue'],'w'); + + %write instructions for launching a job on the cluster + fprintf(fid,'#!/bin/sh\n'); + if mem_debug==0, + fprintf(fid,'mpirun -np %i %s/issm.exe %s %s %s.bin %s.outbin %s.lock 2> %s.errlog >%s.outlog & ',md.np,md.codepath,EnumToString(analysis_type),md.executionpath,modelname,modelname,modelname,modelname,modelname); + else + %fprintf(fid,'LD_PRELOAD=%s mpirun -np %i %s --leak-check=full --gen-suppressions=all --suppressions=%s %s/issm.exe %s %s %s.bin %s.outbin %s.lock 2> %s.errlog >%s.outlog & ',md.valgrindlib,md.np,md.valgrind,md.valgrindsup,md.codepath,EnumToString(analysis_type),md.executionpath,modelname,modelname,modelname,modelname,modelname); + fprintf(fid,'LD_PRELOAD=%s mpirun -np %i %s --leak-check=full --suppressions=%s %s/issm.exe %s %s %s.bin %s.outbin %s.lock 2> %s.errlog >%s.outlog & ',md.valgrindlib,md.np,md.valgrind,md.valgrindsup, md.codepath,EnumToString(analysis_type),md.executionpath,modelname,modelname,modelname,modelname,modelname); + end + + %close file + fclose(fid); + + end + %}}} + function command=LaunchCommand(md,modelruntimename,modelname)%{{{1 + command=['cd ' md.executionpath ' && rm -rf ./' modelruntimename ' && mkdir ' modelruntimename ' && cd ' modelruntimename ' && mv ../' modelruntimename '.tar.gz ./ && tar -zxf ' modelruntimename '.tar.gz && source ' modelname '.queue ']; + end + %}}} + end +end Index: /issm/trunk/src/m/clusters/gemini.m =================================================================== --- /issm/trunk/src/m/clusters/gemini.m (revision 5954) +++ /issm/trunk/src/m/clusters/gemini.m (revision 5954) @@ -0,0 +1,73 @@ +%GEMINI class definition +% +% Usage: +% cluster=gemini(); +% +classdef gemini + properties (SetAccess=public) + %{{{1 + name='gemini' + np =50; %number of processors + queue='debug'; + time=60; + codepath=/workg/edw/larour/issm-2.0/bin + executionpath=/workg/edw/larour/Testing/Execution + login='larour'; + port=0; + %}}} + end + methods + function []=disp(md) %{{{1 + % display the object + disp(sprintf('class ''%s'' object ''%s'' = ',class(md),inputname(1))); + disp(sprintf(' name: %s',md.name)); + disp(sprintf(' np: %i',md.np)); + disp(sprintf(' queue: %s',md.queue)); + disp(sprintf(' time: %i',md.time)); + disp(sprintf(' codepath: %s',md.codepath)); + disp(sprintf(' executionpath: %s',md.executionpath)); + end + %}}} + function IsConsistent(md) %{{{1 + + available_queues={'debug','shortg','longg'}; + queue_requirements_time=[60 180 720]; + queue_requirements_np=[50 50 50]; + + QueueRequirements(available_queues,queue_requirements_time,queue_requirements_np,md.queue,md.np,md.time) + end + %}}} + function BuildQueueScript(md,modelname,analysis_type,mem_debug) %{{{1 + + %open file for writing: + fid=fopen([modelname '.queue'],'w'); + + fprintf(fid,'#!/bin/sh\n'); + fprintf(fid,'#PBS -l walltime=%i\n',md.time*60); %walltime is in seconds. + fprintf(fid,'#PBS -N %s\n',modelname); + fprintf(fid,'#PBS -l ncpus=%i\n',md.np); + if ~isempty(queue), + fprintf(fid,'#PBS -q %s\n',md.queue); + end + fprintf(fid,'#PBS -o %s.outlog \n',modelname); + fprintf(fid,'#PBS -e %s.errlog \n',modelname); + + fprintf(fid,'export PBS_O_WORKDIR=%s\n',md.executionpath); + fprintf(fid,'cd $PBS_O_WORKDIR\n'); + fprintf(fid,'export OMP_NUM_THREADS=1\n'); + fprintf(fid,'dplace -s1 -c0-%i mpirun -np %i %s/issm.exe %s %s %s.bin %s.outbin %s.lock',md.np-1,md.np,md.codepath,EnumToString(analysis_type),md.executionpath,modelname,modelname,modelname); + + %close file + fclose(fid); + + end + %}}} + function command=LaunchCommand(md,modelruntimename,modelname)%{{{1 + command=['cd ' md.executionpath ' && rm -rf ./' modelruntimename ' && mkdir ' modelruntimename ' && cd ' modelruntimename ' && mv ../' modelruntimename '.tar.gz ./ && tar -zxf ' modelruntimename '.tar.gz && qsub ' modelname '.queue ']; + end + %}}} + end +end + + + Index: /issm/trunk/src/m/clusters/larsen.m =================================================================== --- /issm/trunk/src/m/clusters/larsen.m (revision 5954) +++ /issm/trunk/src/m/clusters/larsen.m (revision 5954) @@ -0,0 +1,66 @@ +%LARSEN class definition +% +% Usage: +% cluster=larsen(); +% +classdef larsen + properties (SetAccess=public) + %{{{1 + name='larsen' + np =7; %number of processors + codepath=[issmdir '/bin']; + executionpath=[issmdir '../execution']; + valgrind=[issmdir '/externalpackages/valgrind/install/bin/valgrind']; + valgrindlib=[issmdir '/externalpackages/valgrind/install/lib/libmpidebug.so']; + valgrindsup=[issmdir 'externalpackages/valgrind/issm.supp']; + login='larour'; + port=0; + %}}} + end + methods + function []=disp(md) %{{{1 + % display the object + disp(sprintf('class ''%s'' object ''%s'' = ',class(md),inputname(1))); + disp(sprintf(' name: %s',md.name)); + disp(sprintf(' np: %i',md.np)); + disp(sprintf(' codepath: %s',md.codepath)); + disp(sprintf(' executionpath: %s',md.executionpath)); + disp(sprintf(' valgrind: %s',md.valgrind)); + disp(sprintf(' valgrindlib: %s',md.valgrindlib)); + disp(sprintf(' valgrindsup: %s',md.valgrindsup)); + end + %}}} + function IsConsistent(md) %{{{1 + if md.np>8, + error('IsConsistent error message: number of processors should be lest than 16!'); + end + if isnan(md.np), + error('IsConsistent error message: number of processors should not be NaN!'); + end + end + %}}} + function BuildQueueScript(md,modelname,analysis_type,mem_debug) %{{{1 + + %open file for writing: + fid=fopen([modelname '.queue'],'w'); + + %write instructions for launching a job on the cluster + fprintf(fid,'#!/bin/sh\n'); + if mem_debug==0, + fprintf(fid,'mpirun -np %i %s/issm.exe %s %s %s.bin %s.outbin %s.lock 2> %s.errlog >%s.outlog & ',md.np,md.codepath,EnumToString(analysis_type),md.executionpath,modelname,modelname,modelname,modelname,modelname); + else + %fprintf(fid,'LD_PRELOAD=%s mpirun -np %i %s --leak-check=full --gen-suppressions=all --suppressions=%s %s/issm.exe %s %s %s.bin %s.outbin %s.lock 2> %s.errlog >%s.outlog & ',md.valgrindlib,md.np,md.valgrind,md.valgrindsup,md.codepath,EnumToString(analysis_type),md.executionpath,modelname,modelname,modelname,modelname,modelname); + fprintf(fid,'LD_PRELOAD=%s mpirun -np %i %s --leak-check=full --suppressions=%s %s/issm.exe %s %s %s.bin %s.outbin %s.lock 2> %s.errlog >%s.outlog & ',md.valgrindlib,md.np,md.valgrind,md.valgrindsup, md.codepath,EnumToString(analysis_type),md.executionpath,modelname,modelname,modelname,modelname,modelname); + end + + %close file + fclose(fid); + + end + %}}} + function command=LaunchCommand(md,modelruntimename,modelname)%{{{1 + command=['cd ' md.executionpath ' && rm -rf ./' modelruntimename ' && mkdir ' modelruntimename ' && cd ' modelruntimename ' && mv ../' modelruntimename '.tar.gz ./ && tar -zxf ' modelruntimename '.tar.gz && source ' modelname '.queue ']; + end + %}}} + end +end Index: /issm/trunk/src/m/clusters/none.m =================================================================== --- /issm/trunk/src/m/clusters/none.m (revision 5954) +++ /issm/trunk/src/m/clusters/none.m (revision 5954) @@ -0,0 +1,26 @@ +%NONE class definition +% +% Usage: +% cluster=none(); +% +classdef none + properties (SetAccess=public) + name='none' + end + + methods + function []=disp(md) %{{{1 + % display the object + disp(sprintf('cluster class ''%s'' object ''%s'' = ',class(md),inputname(1))); + disp(sprintf(' name: %s',md.name)); + end + %}}} + function IsConsistent(md) %{{{1 + end + %}}} + function BuildQueueScript(md,modelname,analysis_type,mem_debug) %{{{1 + error('none.BuildQueueScript error message: serial cluster cannot build queue script'); + end + %}}} + end +end Index: /issm/trunk/src/m/clusters/pfe.m =================================================================== --- /issm/trunk/src/m/clusters/pfe.m (revision 5954) +++ /issm/trunk/src/m/clusters/pfe.m (revision 5954) @@ -0,0 +1,100 @@ +%PFE class definition +% +% Usage: +% cluster=pfe(); +% +classdef pfe + properties (SetAccess=public) + %{{{1 + name='pfe' + numnodes=20; + cpuspernode=8; + queue='long'; + time=12*60; + processor='neh'; + codepath='/staff/elarour/trunk/bin'; + executionpath='/nobackupp10/elarour/Testing'; + login='elarour'; + port=1025; + end + properties (SetAccess=private) + np=20*8; + %}}} + end + methods + function []=disp(md) %{{{1 + % display the object + disp(sprintf('class ''%s'' object ''%s'' = ',class(md),inputname(1))); + disp(sprintf(' name: %s',md.name)); + disp(sprintf(' numnodes: %i',md.numnodes)); + disp(sprintf(' cpuspernode: %i',md.cpuspernode)); + disp(sprintf(' np: %i',md.cpuspernode*md.numnodes)); + disp(sprintf(' queue: %s',md.queue)); + disp(sprintf(' time: %i',md.time)); + disp(sprintf(' processor: %s',md.processor)); + disp(sprintf(' codepath: %s',md.codepath)); + disp(sprintf(' executionpath: %s',md.executionpath)); + end + %}}} + function IsConsistent(md) %{{{1 + + available_queues={'long'}; + queue_requirements_time=[7200]; + queue_requirements_np=[2048]; + + QueueRequirements(available_queues,queue_requirements_time,queue_requirements_np,md.queue,md.np,md.time) + + %now, check md.cpuspernode according to processor type + if (strcmpi(md.processor,'har') | strcmpi(md.processor,'neh')), + if ((md.cpuspernode>8 ) | (md.cpuspernode<1)), + error('IsConsistent error message: cpuspernode should be between 1 and 8 for ''neh'' and ''har'' processors'); + end + else if strcmpi(md.processor,'wes'), + if ((md.cpuspernode>12 ) | (md.cpuspernode<1)), + error('IsConsistent error message: cpuspernode should be between 1 and 12 for ''wes'' processors'); + end + else + error('IsConsistent error message: unknown processor type, should be ''neh'',''wes'' or ''har''); + end + + end + %}}} + function BuildQueueScript(md,modelname,analysis_type,mem_debug) %{{{1 +md.np=md.numnodes*md.cpuspernode; + + %open file for writing: + fid=fopen([modelname '.queue'],'w'); + + fprintf(fid,'#PBS -S /bin/bash\n'); + fprintf(fid,'#PBS -N %s\n',modelname); + fprintf(fid,'#PBS -l select=%i:ncpus=%i:model=%s\n',md.numnodes,md.cpuspernode,md.processor); + fprintf(fid,'#PBS -l walltime=%i\n',md.time*60); %walltime is in seconds. + fprintf(fid,'#PBS -W group_list=s1010\n'); + fprintf(fid,'#PBS -m e\n'); + fprintf(fid,'#PBS -o %s.outlog \n',modelname); + fprintf(fid,'#PBS -e %s.errlog \n\n',modelname); + + fprintf(fid,'. /usr/share/modules/init/bash\n\n'); + + fprintf(fid,'module load comp-intel/11.1.046\n'); + fprintf(fid,'module load mpi/mpt.1.25\n'); + fprintf(fid,'module load math/intel_mkl_64_10.0.011\n\n'); + + fprintf(fid,'export PATH="$PATH:."\n\n'); + fprintf(fid,'export MPI_GROUP_MAX=64\n\n'); + + fprintf(fid,'cd $PBS_O_WORKDIR\n\n'); + + fprintf(fid,'mpiexec -verbose -np %i %s/issm.exe %s $PBS_O_WORKDIR %s.bin %s.outbin %s.lock',md.np,md.codepath,EnumToString(analysis_type),modelname,modelname,modelname); + + %close file + fclose(fid); + + end + %}}} + function command=LaunchCommand(md,modelruntimename,modelname)%{{{1 + command=['cd ' md.executionpath ' && rm -rf ./' modelruntimename ' && mkdir ' modelruntimename ' && cd ' modelruntimename ' && mv ../' modelruntimename '.tar.gz ./ && tar -zxf ' modelruntimename '.tar.gz && qsub ' modelname '.queue ']; + end + %}}} + end +end Index: /issm/trunk/src/m/clusters/pollux.m =================================================================== --- /issm/trunk/src/m/clusters/pollux.m (revision 5954) +++ /issm/trunk/src/m/clusters/pollux.m (revision 5954) @@ -0,0 +1,73 @@ +%POLLUX class definition +% +% Usage: +% cluster=pollux(); +% +classdef pollux + properties (SetAccess=public) + %{{{1 + name='pollux' + np =128; %number of processors + queue='shortp'; + time=180; + codepath=/workc/edw/larour/issm-2.0/bin + executionpath=/workc/edw/larour/Testing/Execution + login='larour'; + port=0; + %}}} + end + methods + function []=disp(md) %{{{1 + % display the object + disp(sprintf('class ''%s'' object ''%s'' = ',class(md),inputname(1))); + disp(sprintf(' name: %s',md.name)); + disp(sprintf(' np: %i',md.np)); + disp(sprintf(' queue: %s',md.queue)); + disp(sprintf(' time: %i',md.time)); + disp(sprintf(' codepath: %s',md.codepath)); + disp(sprintf(' executionpath: %s',md.executionpath)); + end + %}}} + function IsConsistent(md) %{{{1 + + available_queues={'shortp','longp'}; + queue_requirements_time=[180 720]; + queue_requirements_np=[128 128]; + + QueueRequirements(available_queues,queue_requirements_time,queue_requirements_np,md.queue,md.np,md.time) + end + %}}} + function BuildQueueScript(md,modelname,analysis_type,mem_debug) %{{{1 + + %open file for writing: + fid=fopen([modelname '.queue'],'w'); + + fprintf(fid,'#!/bin/sh\n'); + fprintf(fid,'#PBS -l walltime=%i\n',md.time*60); %walltime is in seconds. + fprintf(fid,'#PBS -N %s\n',modelname); + fprintf(fid,'#PBS -l ncpus=%i\n',md.np); + if ~isempty(queue), + fprintf(fid,'#PBS -q %s\n',md.queue); + end + fprintf(fid,'#PBS -o %s.outlog \n',modelname); + fprintf(fid,'#PBS -e %s.errlog \n',modelname); + + fprintf(fid,'export PBS_O_WORKDIR=%s\n',md.executionpath); + fprintf(fid,'cd $PBS_O_WORKDIR\n'); + fprintf(fid,'export OMP_NUM_THREADS=1\n'); + fprintf(fid,'dplace -s1 -c0-%i mpirun -np %i %s/issm.exe %s %s %s.bin %s.outbin %s.lock',md.np-1,md.np,md.codepath,EnumToString(analysis_type),md.executionpath,modelname,modelname,modelname); + + %close file + fclose(fid); + + end + %}}} + function command=LaunchCommand(md,modelruntimename,modelname)%{{{1 + command=['cd ' md.executionpath ' && rm -rf ./' modelruntimename ' && mkdir ' modelruntimename ' && cd ' modelruntimename ' && mv ../' modelruntimename '.tar.gz ./ && tar -zxf ' modelruntimename '.tar.gz && qsub ' modelname '.queue ']; + end + %}}} + end +end + + + Index: /issm/trunk/src/m/clusters/wilkes.m =================================================================== --- /issm/trunk/src/m/clusters/wilkes.m (revision 5954) +++ /issm/trunk/src/m/clusters/wilkes.m (revision 5954) @@ -0,0 +1,66 @@ +%WILKES class definition +% +% Usage: +% cluster=wilkes(); +% +classdef wilkes + properties (SetAccess=public) + %{{{1 + name='wilkes' + np =7; %number of processors + codepath=[issmdir '/bin']; + executionpath=[issmdir '../execution']; + valgrind=[issmdir '/externalpackages/valgrind/install/bin/valgrind']; + valgrindlib=[issmdir '/externalpackages/valgrind/install/lib/libmpidebug.so']; + valgrindsup=[issmdir 'externalpackages/valgrind/issm.supp']; + login='larour'; + port=0; + %}}} + end + methods + function []=disp(md) %{{{1 + % display the object + disp(sprintf('class ''%s'' object ''%s'' = ',class(md),inputname(1))); + disp(sprintf(' name: %s',md.name)); + disp(sprintf(' np: %i',md.np)); + disp(sprintf(' codepath: %s',md.codepath)); + disp(sprintf(' executionpath: %s',md.executionpath)); + disp(sprintf(' valgrind: %s',md.valgrind)); + disp(sprintf(' valgrindlib: %s',md.valgrindlib)); + disp(sprintf(' valgrindsup: %s',md.valgrindsup)); + end + %}}} + function IsConsistent(md) %{{{1 + if md.np>8, + error('IsConsistent error message: number of processors should be lest than 16!'); + end + if isnan(md.np), + error('IsConsistent error message: number of processors should not be NaN!'); + end + end + %}}} + function BuildQueueScript(md,modelname,analysis_type,mem_debug) %{{{1 + + %open file for writing: + fid=fopen([modelname '.queue'],'w'); + + %write instructions for launching a job on the cluster + fprintf(fid,'#!/bin/sh\n'); + if mem_debug==0, + fprintf(fid,'mpirun -np %i %s/issm.exe %s %s %s.bin %s.outbin %s.lock 2> %s.errlog >%s.outlog & ',md.np,md.codepath,EnumToString(analysis_type),md.executionpath,modelname,modelname,modelname,modelname,modelname); + else + %fprintf(fid,'LD_PRELOAD=%s mpirun -np %i %s --leak-check=full --gen-suppressions=all --suppressions=%s %s/issm.exe %s %s %s.bin %s.outbin %s.lock 2> %s.errlog >%s.outlog & ',md.valgrindlib,md.np,md.valgrind,md.valgrindsup,md.codepath,EnumToString(analysis_type),md.executionpath,modelname,modelname,modelname,modelname,modelname); + fprintf(fid,'LD_PRELOAD=%s mpirun -np %i %s --leak-check=full --suppressions=%s %s/issm.exe %s %s %s.bin %s.outbin %s.lock 2> %s.errlog >%s.outlog & ',md.valgrindlib,md.np,md.valgrind,md.valgrindsup, md.codepath,EnumToString(analysis_type),md.executionpath,modelname,modelname,modelname,modelname,modelname); + end + + %close file + fclose(fid); + + end + %}}} + function command=LaunchCommand(md,modelruntimename,modelname)%{{{1 + command=['cd ' md.executionpath ' && rm -rf ./' modelruntimename ' && mkdir ' modelruntimename ' && cd ' modelruntimename ' && mv ../' modelruntimename '.tar.gz ./ && tar -zxf ' modelruntimename '.tar.gz && source ' modelname '.queue ']; + end + %}}} + end +end Index: /issm/trunk/src/m/model/LaunchQueueJob.m =================================================================== --- /issm/trunk/src/m/model/LaunchQueueJob.m (revision 5954) +++ /issm/trunk/src/m/model/LaunchQueueJob.m (revision 5954) @@ -0,0 +1,38 @@ +function LaunchQueueJob(md,options) +%LAUNCHQUEUEJOB- Launch job on cluster +% +% Usage: +% LaunchQueueJob(model,options) + +%first, check we have the binary file and the queueing script +if ~exist([ md.name '.bin'],'file'), + error('LaunchQueueJobcosmos error message: Binary input file missing, cannot go forward'); +end + +if ~exist([ md.name '.queue'],'file'), + error('LaunchQueueJobcosmos error message: queueing script missing, cannot go forward'); +end + +if md.qmu_analysis & ~exist([ md.name '.qmu.in'],'file'), + error('LaunchQueueJobcosmos error message: missing dakota input file, cannot go forward'); +end + +if ~strcmpi(options.batch,'yes'), + + %compress the files into one zip. + compressstring=['tar -zcf ' md.runtimename '.tar.gz ' md.name '.bin ' md.name '.queue petsc.rc ']; + if md.qmu_analysis, + compressstring=[compressstring md.name '.qmu.in']; + end + system(compressstring); + + disp('uploading input file and queueing script'); + issmscpout(md.cluster.name,md.cluster.executionpath,md.cluster.login,md.cluster.port,{[md.runtimename '.tar.gz']}); + + disp('launching solution sequence on remote cluster'); + issmssh(md.cluster.name,md.cluster.login,md.cluster.port,LaunchCommand(md.cluster,md.runtimename,md.name)); + +else + disp('batch mode requested: not launching job interactively'); + disp('launch solution sequence on remote cluster by hand'); +end Index: /issm/trunk/src/m/model/ismodelselfconsistent.m =================================================================== --- /issm/trunk/src/m/model/ismodelselfconsistent.m (revision 5953) +++ /issm/trunk/src/m/model/ismodelselfconsistent.m (revision 5954) @@ -167,5 +167,5 @@ end %}}} -%SCALAR +%SCALAR {{{ if ~isscalar(md.control_type), error('model not consistent: md.control_type should be a scalar'); @@ -190,15 +190,5 @@ %}}} %PARALLEL{{{1 -if ~strcmpi(md.cluster,'none'), - - %NAN VALUES - fields={'time','np'}; - checknan(md,fields); - - %FIELD > 0 - fields={'time','np'}; - checkgreaterstrict(md,fields,0); - -end +md.cluster.IsConsistent; %}}} Index: /issm/trunk/src/m/model/loadresultsfromcluster.m =================================================================== --- /issm/trunk/src/m/model/loadresultsfromcluster.m (revision 5953) +++ /issm/trunk/src/m/model/loadresultsfromcluster.m (revision 5954) @@ -5,9 +5,10 @@ % md=loadresultsfromcluster(md,runtimename); -%Get cluster.rc location -cluster_rc_location=which('cluster.rc'); - %Figure out parameters for this particular cluster -[codepath,executionpath,login,port]=ClusterParameters(md.cluster,cluster_rc_location); +codepath=md.cluster.codepath; +executionpath=md.cluster.executionpath; +login=md.cluster.login; +port=md.cluster.port; +clustername=md.cluster.name; if nargin==2, @@ -37,5 +38,5 @@ %copy files from cluster to present directory -issmscpin(md.cluster, login, port, directory, packages); +issmscpin(clustername, login, port, directory, packages); %read log files onto fields @@ -62,5 +63,5 @@ %erase input file if run was carried out on same platform. hostname=oshostname(); -if strcmpi(hostname,md.cluster), +if strcmpi(hostname,clustername), if md.qmu_analysis, delete([['qmu' num2str(GetPId) '/'] md.name '.bin']); Index: /issm/trunk/src/m/model/solve.m =================================================================== --- /issm/trunk/src/m/model/solve.m (revision 5953) +++ /issm/trunk/src/m/model/solve.m (revision 5954) @@ -47,5 +47,5 @@ %If running in parallel, we have a different way of launching the solution %sequences. -if ~strcmpi(md.cluster,'none'), +if ~strcmpi(md.cluster.name,'none'), md=solveparallel(md,options); return; Index: /issm/trunk/src/m/model/solveparallel.m =================================================================== --- /issm/trunk/src/m/model/solveparallel.m (revision 5953) +++ /issm/trunk/src/m/model/solveparallel.m (revision 5954) @@ -10,10 +10,4 @@ md.runtimename=sprintf('%s-%i-%i-%i-%i-%i-%i',md.name,GetPId,c(2),c(3),c(1),c(4),c(5)); -%Get cluster.rc location -cluster_rc_location=which('cluster.rc'); - -%Figure out parameters for this particular cluster -[codepath,executionpath,login,port]=ClusterParameters(md.cluster,cluster_rc_location); - %Marshall model data into a binary file. marshall(md); @@ -25,13 +19,13 @@ %Now, we need to build the queuing script, used by the cluster to launch the job. -BuildQueueingScript(md,executionpath,codepath); +BuildQueueScript(md.cluster,md.name,md.analysis_type,md.mem_debug); %Now, launch the queueing script -md=LaunchQueueJob(md,executionpath,login,port,options); +LaunchQueueJob(md,options); %Do we return, or just wait for results? if (md.waitonlock~=0 & ~strcmpi(options.batch,'yes')), %we wait for the done file - islock=waitonlock(md,executionpath,login,port); + islock=waitonlock(md); if islock==0, %no results to be loaded Index: /issm/trunk/src/m/model/solvers/solversettoasm.m =================================================================== --- /issm/trunk/src/m/model/solvers/solversettoasm.m (revision 5953) +++ /issm/trunk/src/m/model/solvers/solversettoasm.m (revision 5954) @@ -7,3 +7,3 @@ %md.petscoptions={{'mat_type','aij'},{'ksp_type','cgs'},{'pc_type','asm'},{sub_mat_type','mumps'},{'sub_pc_type','lu'},{'pc_asm_overlap',4},{'pc_factor_shift_positive_definite','true'}}; %md.petscoptions={{'mat_type','aij'},{'ksp_type','cgs'},{'pc_type','asm'},{'sub_pc_type','lu'},{'pc_asm_overlap',4}}; -md.petscoptions={{'mat_type','aij'},{'ksp_type','gmres'},{'pc_type','asm'},{'sub_pc_type','lu'},{'pc_asm_overlap',4}}; +md.petscoptions={{'mat_type','aij'},{'ksp_type','gmres'},{'pc_type','asm'},{'sub_pc_type','lu'},{'pc_asm_overlap',3},{'ksp_max_it',100},{'ksp_rtol',1e-30'}}; Index: /issm/trunk/src/m/model/waitonlock.m =================================================================== --- /issm/trunk/src/m/model/waitonlock.m (revision 5953) +++ /issm/trunk/src/m/model/waitonlock.m (revision 5954) @@ -9,7 +9,10 @@ %Get filename (lock file) and options +executionpath=md.cluster.executionpath; +cluster=md.cluster.name; +login=md.cluster.login; +port=md.cluster.port; +timelimit=md.waitonlock; filename=[executionpath '/' md.runtimename '/' md.name '.lock']; -cluster=md.cluster; -timelimit=md.waitonlock; %waitonlock will work if the lock is on the same machine only: