Index: /issm/trunk-jpl/src/m/classes/clusters/stallo.m =================================================================== --- /issm/trunk-jpl/src/m/classes/clusters/stallo.m (revision 23230) +++ /issm/trunk-jpl/src/m/classes/clusters/stallo.m (revision 23230) @@ -0,0 +1,214 @@ +% Stallo cluster class definition +% This is a SLURM queue +% The priorities are given to: +% - Large jobs +% - Short jobs +% - small number of job per user +% +% There are some 20cpu nodes and 16cpu nodes, with 32GB (a few with 128GB) mem per node, you can ask for part of a node if you need more memory.(1 node, 2 CPUS and 10GB per cpu for example) + +% Usage: +% cluster=stallo(); +% cluster=stallo('np',3); +% cluster=stallo('np',3,'login','username'); + +%node task and cpu must be defined, ask andreas for his stallo.py? + +classdef stallo + properties (SetAccess=public) + % {{{ + name = 'stallo'; + login = ''; + numnodes = 2; + cpuspernode = 10; + mem = 1.5; + queue = 'normal'; + time = 2*60; + codepath = ''; + executionpath = ''; + interactive = 0; + port = []; + accountname = ''; + profiling = 0; + bbftp = 0; + % }}} + end + methods + function cluster=stallo(varargin) % {{{ + + %initialize cluster using default settings if provided + if (exist('stallo_settings')==2), stallo_settings; end + + %use provided options to change fields + cluster=AssignObjectFields(pairoptions(varargin{:}),cluster); + end + %}}} + function disp(cluster) % {{{ + % display the object + disp(sprintf('class ''%s'' object ''%s'' = ',class(cluster),inputname(1))); + disp(sprintf(' name: %s',cluster.name)); + disp(sprintf(' login: %s',cluster.login)); + disp(sprintf(' accountname: %s',cluster.accountname)); + disp(sprintf(' numnodes: %i',cluster.numnodes)); + disp(sprintf(' cpuspernode: %i, cpu per nodes',cluster.cpuspernode)); + disp(sprintf(' queue: %s, name of the queue (normal (D), short,singlenode,multinode,devel)',cluster.queue)); + disp(sprintf(' codepath: %s',cluster.codepath)); + disp(sprintf(' executionpath: %s',cluster.executionpath)); + disp(sprintf(' interactive: %i',cluster.interactive)); + disp(sprintf(' time: %i, walltime requested in minutes',cluster.time)); + disp(sprintf(' memory: %i, memory per CPU',cluster.mem)); + disp(sprintf(' profiling: %i, enable profiling if 1 default is 0',cluster.profiling)); + end + %}}} + function md = checkconsistency(cluster,md,solution,analyses) % {{{ + + available_queues={'short','normal','singlenode','multinode','devel'}; + queue_requirements_time=[60 2*24*60 28*24*60 28*24*60 4*60]; + queue_requirements_np=[2048 2048 20 2048 2048]; + + QueueRequirements(available_queues,queue_requirements_time,queue_requirements_np,cluster.queue,cluster.np,1) + + %Miscelaneous + if isempty(cluster.login), md = checkmessage(md,'login empty'); end + if isempty(cluster.accountname), md = checkmessage(md,'accountname empty'); end + if isempty(cluster.codepath), md = checkmessage(md,'codepath empty'); end + if isempty(cluster.executionpath), md = checkmessage(md,'executionpath empty'); end + + end + %}}} + function numprocs=np(cluster) % {{{ + %compute number of processors + numprocs=cluster.numnodes*cluster.cpuspernode; + end + %}}} + function BuildQueueScript(cluster,dirname,modelname,solution,io_gather,isvalgrind,isgprof,isdakota,isoceancoupling) % {{{ + + if(isvalgrind), disp('valgrind not supported by cluster, ignoring...'); end + if(isgprof), disp('gprof not supported by cluster, ignoring...'); end + + executable='issm.exe'; + if isdakota, + version=IssmConfig('_DAKOTA_VERSION_'); version=str2num(version(1:3)); + if (version>=6), + executable='issm_dakota.exe'; + end + end + if isoceancoupling, + executable='issm_ocean.exe'; + end + + %write queuing script + fid=fopen([modelname '.queue'],'w'); + fprintf(fid,'#!/bin/bash -l\n'); + fprintf(fid,'#SBATCH --job-name=%s \n',modelname); + fprintf(fid,'#SBATCH --qos %s \n',cluster.queue); + fprintf(fid,'#SBATCH --nodes=%i \n',cluster.numnodes); + fprintf(fid,'#SBATCH --ntasks-per-node=%i \n',cluster.cpuspernode); + fprintf(fid,'#SBATCH --time=%02i-%02i:%02i:00 \n',floor(cluster.time/(3600*24)),floor(cluster.time/3600),floor(mod(cluster.time,60))); + fprintf(fid,'#SBATCH --mem-per-cpu=%iMB\n',1000*cluster.mem); ... + % mem in MB + if (mod(cluster.np,16)+mod(cluster.np,20))==0, + fprintf(fid,'#SBATCH --ntask=%i\n',cluster.np); + end + fprintf(fid,'#SBATCH --account=%s\n',cluster.accountname); + fprintf(fid,'#SBATCH --output %s/%s/%s.outlog \n',cluster.executionpath,dirname,modelname); + fprintf(fid,'#SBATCH --error %s/%s/%s.errlog \n\n',cluster.executionpath,dirname,modelname); + fprintf(fid,'export ISSM_DIR="%s/../"\n',cluster.codepath);%FIXME + fprintf(fid,'module purge\n'); + fprintf(fid,'module load CMake/3.8.0-GCCcore-6.3.0\n'); + fprintf(fid,'module load Automake/1.15.1-GCCcore-6.3.0\n'); + fprintf(fid,'module load libtool/2.4.6\n'); + fprintf(fid,'module load OpenSSL/1.1.0e-intel-2017a\n'); + fprintf(fid,'module load PETSc/3.7.5-intel-2017a-downloaded-deps\n'); + + fprintf(fid,'cd %s/%s/\n\n',cluster.executionpath,dirname); + if cluster.profiling==1, + fprintf(fid,'module load perf-report\n'); + fprintf(fid,'perf-report mpirun -np %i %s/%s %s %s/%s %s\n',cluster.np,cluster.codepath,executable,solution,cluster.executionpath,dirname,modelname); + else + fprintf(fid,'mpirun -np %i %s/%s %s %s/%s %s\n',cluster.np,cluster.codepath,executable,solution,cluster.executionpath,dirname,modelname); + end + %}}} + if ~io_gather, %concatenate the output files: + fprintf(fid,'cat %s.outbin.* > %s.outbin',modelname,modelname); + end + fclose(fid); + + %in interactive mode, create a run file, and errlog and outlog file + if cluster.interactive, + fid=fopen([modelname '.run'],'w'); + fprintf(fid,'mpiexec -np %i %s/issm.exe %s %s %s\n',cluster.np,cluster.codepath,solution,[cluster.executionpath '/' dirname],modelname); + if ~io_gather, %concatenate the output files: + fprintf(fid,'cat %s.outbin.* > %s.outbin',modelname,modelname); + end + fclose(fid); + fid=fopen([modelname '.errlog'],'w'); + fclose(fid); + fid=fopen([modelname '.outlog'],'w'); + fclose(fid); + end + end + + + function UploadQueueJob(cluster,modelname,dirname,filelist)% {{{ + + %compress the files into one zip. + compressstring=['tar -zcf ' dirname '.tar.gz ']; + for i=1:numel(filelist), + compressstring = [compressstring ' ' filelist{i}]; + end + if cluster.interactive, + compressstring = [compressstring ' ' modelname '.run ' modelname '.errlog ' modelname '.outlog ']; + end + system(compressstring); + disp('uploading input file and queueing script'); + if cluster.interactive==10, + directory=[pwd() '/run/']; + elseif cluster.interactive, + directory=[cluster.executionpath '/Interactive' num2str(cluster.interactive)]; + else + directory=cluster.executionpath; + end + + if ~cluster.bbftp, + issmscpout(cluster.name,directory,cluster.login,cluster.port,{[dirname '.tar.gz']}); + else + issmbbftpout(cluster.name,directory,cluster.login,cluster.port,cluster.numstreams,{[dirname '.tar.gz']}); + + end + end %}}} + function LaunchQueueJob(cluster,modelname,dirname,filelist,restart,batch)% {{{ + + %lauch command, to be executed via ssh + if ~cluster.interactive, + if ~isempty(restart) + launchcommand=['cd ' cluster.executionpath ' && cd ' dirname ' && qsub ' modelname '.queue ']; + else + launchcommand=['cd ' cluster.executionpath ' && rm -rf ./' dirname ' && mkdir ' dirname ... + ' && cd ' dirname ' && mv ../' dirname '.tar.gz ./ && tar -zxf ' dirname '.tar.gz && qsub ' modelname '.queue ']; + end + else + if ~isempty(restart) + launchcommand=['cd ' cluster.executionpath '/Interactive' num2str(cluster.interactive)]; + else + if cluster.interactive==10, + launchcommand=['cd ' pwd() '/run && tar -zxf ' dirname '.tar.gz']; + else + launchcommand=['cd ' cluster.executionpath '/Interactive' num2str(cluster.interactive) ' && tar -zxf ' dirname '.tar.gz']; + end + end + end + + disp('launching solution sequence on remote cluster'); + issmssh(cluster.name,cluster.login,cluster.port,launchcommand); + end + %}}} + function Download(cluster,dirname,filelist)% {{{ + + %copy files from cluster to current directory + directory=[cluster.executionpath '/' dirname '/']; + issmscpin(cluster.name,cluster.login,cluster.port,directory,filelist); + + end %}}} + end +end Index: /issm/trunk-jpl/src/m/classes/clusters/stallo.py =================================================================== --- /issm/trunk-jpl/src/m/classes/clusters/stallo.py (revision 23229) +++ /issm/trunk-jpl/src/m/classes/clusters/stallo.py (revision 23230) @@ -51,5 +51,5 @@ #OK get other fields self=options.AssignObjectFields(self) - self.np=self.numnodes*self.cpuspernode + self.np=self.numnodes*self.cpuspernode # }}} @@ -61,5 +61,5 @@ s = "%s\n%s"%(s,fielddisplay(self,'login','login')) s = "%s\n%s"%(s,fielddisplay(self,'numnodes','number of nodes')) - s = "%s\n%s"%(s,fielddisplay(self,'cpuspernode','number of nodes per CPUs')) + s = "%s\n%s"%(s,fielddisplay(self,'cpuspernode','number of CPUs per nodes')) s = "%s\n%s"%(s,fielddisplay(self,'mem','memory per CPU')) s = "%s\n%s"%(s,fielddisplay(self,'queue','name of the queue (normal (D), short,singlenode,multinode,devel)')) @@ -120,7 +120,7 @@ h,m=divmod(m, 60) d,h=divmod(h, 60) - timestring="%02d-%02d:%02d:%02d" % (d, h, m, s) - fid.write('#SBATCH --time=%s\n' % timestring) #walltime is minutes - fid.write('#SBATCH --mem-per-cpu=%iGB\n' % self.mem)# mem is in GB +# timestring="%02d-%02d:%02d:%02d" % (d, h, m, s) + fid.write('#SBATCH --time={}\n'.format(timestring)) #walltime is minutes + fid.write('#SBATCH --mem-per-cpu={}MB\n'.format(1000*self.mem))# mem is in MB (to keep it integral) if (np.mod(self.np,16)+np.mod(self.np,20))==0: fid.write('#SBATCH --ntask=%i\n' % self.np)