Index: /issm/trunk-jpl/src/m/classes/clusters/cyclone.py =================================================================== --- /issm/trunk-jpl/src/m/classes/clusters/cyclone.py (revision 20784) +++ /issm/trunk-jpl/src/m/classes/clusters/cyclone.py (revision 20784) @@ -0,0 +1,125 @@ +import subprocess +from fielddisplay import fielddisplay +from EnumToString import EnumToString +from pairoptions import pairoptions +from issmssh import issmssh +from issmscpin import issmscpin +from issmscpout import issmscpout +from QueueRequirements import QueueRequirements +import datetime +try: + from cyclone_settings import cyclone_settings +except ImportError: + print 'You need cyclone_settings.py to proceed, check presence and sys.path' + +class cyclone(object): + """ + Be aware that this is not a cluster as we usually know them. There is no scheduling and ressources are pretty low. + The Computer have 20 cpus and 512Gb of memory used by a number of person so be respectful with your usage. + I putted some restrictive upper limits to avoid over-use. (Basile) + + Usage: + cluster=cyclone(); + """ + + def __init__(self,*args): + # {{{ + self.name = 'cyclone' + self.login = '' + self.np = 2 + self.time = 100 + self.codepath = '' + self.executionpath = '' + self.port = '' + self.interactive = 0 + + #use provided options to change fields + options=pairoptions(*args) + + #initialize cluster using user settings if provided + self=cyclone_settings(self) + #OK get other fields + self=options.AssignObjectFields(self) + + # }}} + + def __repr__(self): + # {{{ + # display the object + s = "class cyclone object:" + s = "%s\n%s"%(s,fielddisplay(self,'name','name of the cluster')) + s = "%s\n%s"%(s,fielddisplay(self,'login','login')) + s = "%s\n%s"%(s,fielddisplay(self,'np','number of processes')) + s = "%s\n%s"%(s,fielddisplay(self,'time','walltime requested in minutes')) + s = "%s\n%s"%(s,fielddisplay(self,'codepath','code path on the cluster')) + s = "%s\n%s"%(s,fielddisplay(self,'executionpath','execution path on the cluster')) + return s + # }}} + + def checkconsistency(self,md,solution,analyses): + # {{{ + #Miscelaneous + if not self.login: + md = md.checkmessage('login empty') + if not self.codepath: + md = md.checkmessage('codepath empty') + if not self.executionpath: + md = md.checkmessage('executionpath empty') + if self.time>72: + md = md.checkmessage('walltime exceeds 72h for niceness this is not allowed, if you need more time consider shifting to one of the Notur systems') + if self.np >10: + md = md.checkmessage('number of process excess 10, if you need more processing power consider shifting to one of the Notur systems') + + return self + # }}} + def BuildQueueScript(self,dirname,modelname,solution,io_gather,isvalgrind,isgprof,isdakota): + # {{{ + + executable='issm.exe' + + #write queuing script + shortname=modelname[0:min(12,len(modelname))] + fid=open(modelname+'.queue','w') + fid.write('export ISSM_DIR="%s/../"\n' % self.codepath) + fid.write('source $ISSM_DIR/etc/environment.sh\n') + fid.write('INTELLIBS="/opt/intel/intelcompiler-12.04/composerxe-2011.4.191/compiler/lib/intel64"\n') + fid.write('export LD_LIBRARY_PATH=$LD_LIBRARY_PATH:/usr/lib/:$INTELLIBS\n') + fid.write('export CPLUS_INCLUDE_PATH=$CPLUS_INCLUDE_PATH:/usr/include/x86_64-linux-gnu/c++/4.8\n') + fid.write('cd %s/%s/\n\n' % (self.executionpath,dirname)) + rundir=self.executionpath+'/'+dirname + runfile=self.executionpath+'/'+dirname+'/'+modelname + fid.write('mpiexec -np %i %s/%s %s %s %s >%s.outlog 2>%s.errlog\n' % (self.np,self.codepath,executable,str(EnumToString(solution)[0]),rundir,modelname,runfile,runfile)) + fid.close() + + # }}} + def UploadQueueJob(self,modelname,dirname,filelist): + # {{{ + + #compress the files into one zip. + compressstring='tar -zcf %s.tar.gz ' % dirname + for file in filelist: + compressstring += ' %s' % file + subprocess.call(compressstring,shell=True) + + print 'uploading input file and queueing script' + issmscpout(self.name,self.executionpath,self.login,self.port,[dirname+'.tar.gz']) + + # }}} + def LaunchQueueJob(self,modelname,dirname,filelist,restart,batch): + # {{{ + + print 'launching solution sequence on remote cluster' + if restart: + launchcommand='cd %s && cd %s && qsub %s.queue' % (self.executionpath,dirname,modelname) + else: + launchcommand='cd %s && rm -rf ./%s && mkdir %s && cd %s && mv ../%s.tar.gz ./ && tar -zxf %s.tar.gz && chmod +x ./%s.queue && ./%s.queue' % (self.executionpath,dirname,dirname,dirname,dirname,dirname,modelname,modelname) + issmssh(self.name,self.login,self.port,launchcommand) + + # }}} + def Download(self,dirname,filelist): + # {{{ + + #copy files from cluster to current directory + directory='%s/%s/' % (self.executionpath,dirname) + issmscpin(self.name,self.login,self.port,directory,filelist) + # }}} Index: /issm/trunk-jpl/src/m/classes/clusters/hexagon.py =================================================================== --- /issm/trunk-jpl/src/m/classes/clusters/hexagon.py (revision 20784) +++ /issm/trunk-jpl/src/m/classes/clusters/hexagon.py (revision 20784) @@ -0,0 +1,147 @@ +import subprocess +from fielddisplay import fielddisplay +from EnumToString import EnumToString +from pairoptions import pairoptions +from issmssh import issmssh +from issmscpin import issmscpin +from issmscpout import issmscpout +from QueueRequirements import QueueRequirements +import datetime +try: + from hexagon_settings import hexagon_settings +except ImportError: + print 'You need hexagon_settings.py to proceed, check presence and sys.path' + +class hexagon(object): + """ + Hexagon cluster class definition + Hexagon have nodes built of 2*16 CPUs. Nodes are dedicated to one job so the best usage is to use 32 procs per nodes (16 per cores) as it is what is billed anyway. + You can reduce this number if you run out of memory as the total node memory is divided by the number of procs + Usage: + cluster=hexagon(); + """ + + def __init__(self,*args): + # {{{ + self.name = 'hexagon' + self.login = '' + self.numnodes = 2 + self.procspernodes = 32 + self.mem = 32000 + self.queue = 'batch' + self.time = 2*60 + self.codepath = '' + self.executionpath = '' + self.interactive = 0 + self.port = [] + self.accountname = '' + + #use provided options to change fields + options=pairoptions(*args) + + #initialize cluster using user settings if provided + self=hexagon_settings(self) + self.np=self.numnodes*self.procspernodes + #OK get other fields + self=options.AssignObjectFields(self) + + # }}} + def __repr__(self): + # {{{ + # display the object + s = "class hexagon object:" + s = "%s\n%s"%(s,fielddisplay(self,'name','name of the cluster')) + s = "%s\n%s"%(s,fielddisplay(self,'login','login')) + s = "%s\n%s"%(s,fielddisplay(self,'numnodes','number of nodes')) + s = "%s\n%s"%(s,fielddisplay(self,'procspernodes','number of mpi procs per nodes default and optimal is 32')) + s = "%s\n%s"%(s,fielddisplay(self,'mem','Total node memory')) + s = "%s\n%s"%(s,fielddisplay(self,'queue','name of the queue')) + s = "%s\n%s"%(s,fielddisplay(self,'time','walltime requested in minutes')) + s = "%s\n%s"%(s,fielddisplay(self,'codepath','code path on the cluster')) + s = "%s\n%s"%(s,fielddisplay(self,'executionpath','execution path on the cluster')) + s = "%s\n%s"%(s,fielddisplay(self,'interactive','')) + s = "%s\n%s"%(s,fielddisplay(self,'accountname','your cluster account')) + return s + # }}} + def checkconsistency(self,md,solution,analyses): + # {{{ + #mem should not be over 32000mb + #numprocs should not be over 4096 + #we have cpupernodes*numberofcpus=mppwidth and mppnppn=cpupernodes, + #Miscelaneous + if not self.login: + md = md.checkmessage('login empty') + if not self.codepath: + md = md.checkmessage('codepath empty') + if not self.executionpath: + md = md.checkmessage('executionpath empty') + if self.interactive==1: + md = md.checkmessage('interactive mode not implemented') + if self.mem>32000: + md = md.checkmessage('asking too much memory max is 32000 per node') + return self + # }}} + def BuildQueueScript(self,dirname,modelname,solution,io_gather,isvalgrind,isgprof,isdakota): + # {{{ + + executable='issm.exe' + if isdakota: + version=IssmConfig('_DAKOTA_VERSION_')[0:2] + version=float(version) + if version>=6: + executable='issm_dakota.exe' + + #write queuing script + shortname=modelname[0:min(12,len(modelname))] + fid=open(modelname+'.queue','w') + fid.write('#!/bin/bash\n') + fid.write('#PBS -N %s \n' % shortname) + fid.write('#PBS -l mppwidth=%i,mppnppn=%i\n' % (self.np,self.procspernodes)) + timestring= str(datetime.timedelta(minutes=self.time)) + fid.write('#PBS -l walltime=%s\n' % timestring) #walltime is hh:mm:ss + fid.write('#PBS -l mppmem=%imb\n' % int(self.mem/self.procspernodes)) + fid.write('#PBS -A %s\n' % self.accountname) + fid.write('#PBS -o %s/%s/%s.outlog \n' % (self.executionpath,dirname,modelname)) + fid.write('#PBS -e %s/%s/%s.errlog \n\n' % (self.executionpath,dirname,modelname)) + fid.write('export ISSM_DIR="%s/../"\n' % self.codepath) + fid.write('export CRAY_ROOTFS=DSL\n') + fid.write('module swap PrgEnv-cray/5.2.40 PrgEnv-gnu\n') + fid.write('module load cray-petsc\n') + fid.write('module load cray-tpsl\n') + fid.write('module load cray-mpich\n') + fid.write('module load gsl\n') + fid.write('cd %s/%s/\n\n' % (self.executionpath,dirname)) + fid.write('aprun -B %s/%s %s %s/%s %s\n' % (self.codepath,executable,str(EnumToString(solution)[0]),self.executionpath,dirname,modelname)) + fid.close() + + # }}} + def UploadQueueJob(self,modelname,dirname,filelist): + # {{{ + + #compress the files into one zip. + compressstring='tar -zcf %s.tar.gz ' % dirname + for file in filelist: + compressstring += ' %s' % file + subprocess.call(compressstring,shell=True) + + print 'uploading input file and queueing script' + issmscpout(self.name,self.executionpath,self.login,self.port,[dirname+'.tar.gz']) + + # }}} + def LaunchQueueJob(self,modelname,dirname,filelist,restart,batch): + # {{{ + + print 'launching solution sequence on remote cluster' + if restart: + launchcommand='cd %s && cd %s && qsub %s.queue' % (self.executionpath,dirname,modelname) + else: + launchcommand='cd %s && rm -rf ./%s && mkdir %s && cd %s && mv ../%s.tar.gz ./ && tar -zxf %s.tar.gz && qsub %s.queue' % (self.executionpath,dirname,dirname,dirname,dirname,dirname,modelname) + issmssh(self.name,self.login,self.port,launchcommand) + + # }}} + def Download(self,dirname,filelist): + # {{{ + #copy files from cluster to current directory + directory='%s/%s/' % (self.executionpath,dirname) + issmscpin(self.name,self.login,self.port,directory,filelist) + # }}} Index: /issm/trunk-jpl/src/m/classes/clusters/vilje.m =================================================================== --- /issm/trunk-jpl/src/m/classes/clusters/vilje.m (revision 20783) +++ /issm/trunk-jpl/src/m/classes/clusters/vilje.m (revision 20784) @@ -9,10 +9,10 @@ properties (SetAccess=public) % {{{ - name = 'vilje'; + name = 'vilje'; login = ''; numnodes = 2; cpuspernode = 32; - procspernodes = 16; - mem = 28; + procspernodes = 16; + mem = 28; queue = 'workq'; time = 2*60; @@ -20,7 +20,6 @@ executionpath = ''; interactive = 0; - port = []; - accountname = ''; - + port = []; + accountname = ''; end @@ -44,9 +43,8 @@ disp(sprintf(' name: %s',cluster.name)); disp(sprintf(' login: %s',cluster.login)); - disp(sprintf(' accountname: %s',cluster.accountname)); + disp(sprintf(' accountname: %s',cluster.accountname)); disp(sprintf(' numnodes: %i',cluster.numnodes)); disp(sprintf(' cpuspernode: %i',cluster.cpuspernode)); - disp(sprintf(' np: %i', ... - cluster.cpuspernode*cluster.numnodes)); + disp(sprintf(' np: %i', cluster.cpuspernode*cluster.numnodes)); disp(sprintf(' procspernodes: %i',cluster.procspernodes)); disp(sprintf(' queue: %s',cluster.queue)); @@ -68,5 +66,5 @@ %Miscelaneous if isempty(cluster.login), md = checkmessage(md,'login empty'); end - if isempty(cluster.accountname), md = checkmessage(md,'accountname empty'); end + if isempty(cluster.accountname), md = checkmessage(md,'accountname empty'); end if isempty(cluster.codepath), md = checkmessage(md,'codepath empty'); end if isempty(cluster.executionpath), md = checkmessage(md,'executionpath empty'); end @@ -86,10 +84,7 @@ fprintf(fid,'#PBS -S /bin/bash\n'); fprintf(fid,'#PBS -N %s\n',modelname); - fprintf(fid,'#PBS -l select=%i:ncpus=%i:mpiprocs=%i\n',cluster.numnodes,cluster.cpuspernode,16); - fprintf(fid,'#PBS -l walltime=%s\n',cluster.time); %walltime is in seconds. - fprintf(fid,'#PBS -A %s\n',cluster.accountname); -% fprintf(fid,'#PBS -q %s \n',cluster.queue); -% fprintf(fid,'#PBS -l mem=%igb\n',cluster.memory); -% fprintf(fid,'#PBS -M mmorligh@uci.edu\n'); + fprintf(fid,'#PBS -l select=%i:ncpus=%i:mpiprocs=%i\n',cluster.numnodes,cluster.cpuspernode,16); + fprintf(fid,'#PBS -l walltime=%s\n',cluster.time); %walltime is in seconds. + fprintf(fid,'#PBS -A %s\n',cluster.accountname); fprintf(fid,'#PBS -o %s.outlog \n',modelname); fprintf(fid,'#PBS -e %s.errlog \n\n',modelname); @@ -118,8 +113,6 @@ %compute number of processors - cluster.np=cluster.numnodes*cluster.cpuspernode; - - - shortname = substring(modelname,1,min(12,length(modelname))); + cluster.np=cluster.numnodes*cluster.cpuspernode; + shortname = substring(modelname,1,min(12,length(modelname))); %write queuing script @@ -128,14 +121,13 @@ fprintf(fid,'#PBS -N %s\n',shortname); fprintf(fid,'#PBS -q %s \n',cluster.queue); - fprintf(fid,'#PBS -l select=%i:ncpus=%i:mpiprocs=%i\n',cluster.numnodes,cluster.cpuspernode,cluster.procspernodes); + fprintf(fid,'#PBS -l select=%i:ncpus=%i:mpiprocs=%i\n',cluster.numnodes,cluster.cpuspernode,cluster.procspernodes); fprintf(fid,'#PBS -l walltime=%s\n',duration(0,cluster.time,0)); %walltime is in minutes. - fprintf(fid,'#PBS -A %s\n',cluster.accountname); + fprintf(fid,'#PBS -A %s\n',cluster.accountname); fprintf(fid,'#PBS -o %s.outlog \n',[cluster.executionpath '/' dirname '/' modelname]); fprintf(fid,'#PBS -e %s.errlog \n\n',[cluster.executionpath '/' dirname '/' modelname]); fprintf(fid,'export ISSM_DIR="%s/../"\n',cluster.codepath); %FIXME fprintf(fid,'source $ISSM_DIR/etc/environment.sh\n'); %FIXME - fprintf(fid,'cd %s/%s\n\n', ... - cluster.executionpath,dirname); - fprintf(fid,'mpiexec -np %i %s/%s %s %s %s\n',cluster.np,cluster.codepath,executable,EnumToString(solution),[cluster.executionpath '/' dirname],modelname); + fprintf(fid,'cd %s/%s\n\n',cluster.executionpath,dirname); + fprintf(fid,'mpiexec -np %i %s/%s %s %s %s\n',cluster.np,cluster.codepath,executable,EnumToString(solution),[cluster.executionpath '/' dirname],modelname); if ~io_gather, %concatenate the output files: @@ -167,13 +159,8 @@ compressstring = [compressstring ' ' filelist{i}]; end - system(compressstring); - disp('uploading input file and queueing script'); - directory=cluster.executionpath; - - issmbbftpout(cluster.name,directory, ... - cluster.login,cluster.port,cluster.numstreams,{[dirname '.tar.gz']}); + issmbbftpout(cluster.name,directory,cluster.login,cluster.port,cluster.numstreams,{[dirname '.tar.gz']}); end @@ -188,4 +175,5 @@ issmssh(cluster.name,cluster.login,cluster.port,launchcommand); end %}}} + function Download(cluster,dirname,filelist)% {{{ Index: /issm/trunk-jpl/src/m/classes/clusters/vilje.py =================================================================== --- /issm/trunk-jpl/src/m/classes/clusters/vilje.py (revision 20783) +++ /issm/trunk-jpl/src/m/classes/clusters/vilje.py (revision 20784) @@ -27,6 +27,6 @@ self.numnodes = 2 self.cpuspernode = 32 - self.procspernodes = 16 - self.mem = 28 + self.procspernodes = 16 + self.mem = 28 self.queue = 'workq' self.time = 2*60 @@ -34,6 +34,6 @@ self.executionpath = '' self.interactive = 0 - self.port = [] - self.accountname = '' + self.port = [] + self.accountname = '' #use provided options to change fields @@ -95,5 +95,5 @@ #write queuing script - shortname=modelname[0:min(12,len(modelname))] + shortname=modelname[0:min(12,len(modelname))] fid=open(modelname+'.queue','w') fid.write('#PBS -S /bin/bash\n') @@ -107,6 +107,4 @@ fid.write('#PBS -o %s/%s/%s.outlog \n' % (self.executionpath,dirname,modelname)) fid.write('#PBS -e %s/%s/%s.errlog \n\n' % (self.executionpath,dirname,modelname)) - fid.write('export PATH="$PATH:."\n\n') - fid.write('export MPI_GROUP_MAX=64\n\n') fid.write('export ISSM_DIR="%s/../"\n' % self.codepath) fid.write('source $ISSM_DIR/etc/environment.sh\n') @@ -117,10 +115,10 @@ # }}} def UploadQueueJob(self,modelname,dirname,filelist): - # {{{ + # {{{ #compress the files into one zip. compressstring='tar -zcf %s.tar.gz ' % dirname for file in filelist: - compressstring += ' %s' % file + compressstring += ' %s' % file subprocess.call(compressstring,shell=True) @@ -130,5 +128,5 @@ # }}} def LaunchQueueJob(self,modelname,dirname,filelist,restart,batch): - # {{{ + # {{{ print 'launching solution sequence on remote cluster'