| 1 | import socket
|
|---|
| 2 | import os
|
|---|
| 3 | import math
|
|---|
| 4 | import subprocess
|
|---|
| 5 | from IssmConfig import IssmConfig
|
|---|
| 6 | from EnumToString import EnumToString
|
|---|
| 7 | from issmdir import issmdir
|
|---|
| 8 | from pairoptions import pairoptions
|
|---|
| 9 | from issmssh import issmssh
|
|---|
| 10 | from issmscpin import issmscpin
|
|---|
| 11 | from issmscpout import issmscpout
|
|---|
| 12 | import MatlabFuncs as m
|
|---|
| 13 |
|
|---|
| 14 | class generic(object):
|
|---|
| 15 | """
|
|---|
| 16 | GENERIC cluster class definition
|
|---|
| 17 |
|
|---|
| 18 | Usage:
|
|---|
| 19 | cluster=generic('name','astrid','np',3);
|
|---|
| 20 | cluster=generic('name',oshostname(),'np',3,'login','username');
|
|---|
| 21 | """
|
|---|
| 22 |
|
|---|
| 23 | def __init__(self,**kwargs): # {{{
|
|---|
| 24 |
|
|---|
| 25 | self.name=''
|
|---|
| 26 | self.login=''
|
|---|
| 27 | self.np=1
|
|---|
| 28 | self.port=0
|
|---|
| 29 | self.interactive=1
|
|---|
| 30 | self.codepath=issmdir()+'/bin'
|
|---|
| 31 | self.executionpath=issmdir()+'/execution'
|
|---|
| 32 | self.valgrind=issmdir()+'/externalpackages/valgrind/install/bin/valgrind'
|
|---|
| 33 | self.valgrindlib=issmdir()+'/externalpackages/valgrind/install/lib/libmpidebug.so'
|
|---|
| 34 | self.valgrindsup=issmdir()+'/externalpackages/valgrind/issm.supp'
|
|---|
| 35 |
|
|---|
| 36 | #use provided options to change fields
|
|---|
| 37 | options=pairoptions(**kwargs)
|
|---|
| 38 |
|
|---|
| 39 | #get name
|
|---|
| 40 | self.name=socket.gethostname()
|
|---|
| 41 |
|
|---|
| 42 | #initialize cluster using user settings if provided
|
|---|
| 43 | if os.path.exists(self.name+'_settings.py'):
|
|---|
| 44 | exec(compile(open(self.name+'_settings.py').read(), self.name+'_settings.py', 'exec'),globals())
|
|---|
| 45 |
|
|---|
| 46 | #OK get other fields
|
|---|
| 47 | self=options.AssignObjectFields(self)
|
|---|
| 48 | # }}}
|
|---|
| 49 | def __repr__(self): # {{{
|
|---|
| 50 | # display the object
|
|---|
| 51 | s ="class '%s' object '%s' = \n" % (type(self),'self')
|
|---|
| 52 | s+=" name: %s\n" % self.name
|
|---|
| 53 | s+=" login: %s\n" % self.login
|
|---|
| 54 | s+=" np: %i\n" % self.np
|
|---|
| 55 | s+=" port: %i\n" % self.port
|
|---|
| 56 | s+=" codepath: %s\n" % self.codepath
|
|---|
| 57 | s+=" executionpath: %s\n" % self.executionpath
|
|---|
| 58 | s+=" valgrind: %s\n" % self.valgrind
|
|---|
| 59 | s+=" valgrindlib: %s\n" % self.valgrindlib
|
|---|
| 60 | s+=" valgrindsup: %s\n" % self.valgrindsup
|
|---|
| 61 | return s
|
|---|
| 62 | # }}}
|
|---|
| 63 | def checkconsistency(self,md,solution,analyses): # {{{
|
|---|
| 64 | if self.np<1:
|
|---|
| 65 | md = checkmessage(md,'number of processors should be at least 1')
|
|---|
| 66 | if math.isnan(self.np):
|
|---|
| 67 | md = checkmessage(md,'number of processors should not be NaN!')
|
|---|
| 68 |
|
|---|
| 69 | return md
|
|---|
| 70 | # }}}
|
|---|
| 71 | def BuildQueueScript(self,dirname,modelname,solution,io_gather,isvalgrind,isgprof,isdakota): # {{{
|
|---|
| 72 |
|
|---|
| 73 | executable='issm.exe';
|
|---|
| 74 | if isdakota:
|
|---|
| 75 | version=IssmConfig('_DAKOTA_VERSION_')[0:2]
|
|---|
| 76 | version=float(version)
|
|---|
| 77 | if version>=6:
|
|---|
| 78 | executable='issm_dakota.exe'
|
|---|
| 79 |
|
|---|
| 80 | #write queuing script
|
|---|
| 81 | if not m.ispc():
|
|---|
| 82 |
|
|---|
| 83 | fid=open(modelname+'.queue','w')
|
|---|
| 84 | fid.write('#!/bin/sh\n')
|
|---|
| 85 | if not isvalgrind:
|
|---|
| 86 | if self.interactive:
|
|---|
| 87 | if IssmConfig('_HAVE_MPI_')[0]:
|
|---|
| 88 | fid.write('mpiexec -np %i %s/%s %s %s/%s %s ' % (self.np,self.codepath,executable,EnumToString(solution)[0],self.executionpath,dirname,modelname))
|
|---|
| 89 | else:
|
|---|
| 90 | fid.write('%s/%s %s %s/%s %s ' % (self.codepath,executable,EnumToString(solution)[0],self.executionpath,dirname,modelname))
|
|---|
| 91 | else:
|
|---|
| 92 | if IssmConfig('_HAVE_MPI_')[0]:
|
|---|
| 93 | fid.write('mpiexec -np %i %s/%s %s %s/%s %s 2> %s.errlog >%s.outlog ' % (self.np,self.codepath,executable,EnumToString(solution)[0],self.executionpath,dirname,modelname,modelname,modelname))
|
|---|
| 94 | else:
|
|---|
| 95 | fid.write('%s/%s %s %s/%s %s 2> %s.errlog >%s.outlog ' % (self.codepath,executable,EnumToString(solution)[0],self.executionpath,dirname,modelname,modelname,modelname))
|
|---|
| 96 | elif isgprof:
|
|---|
| 97 | fid.write('\n gprof %s/%s gmon.out > %s.performance' % (self.codepath,executable,modelname))
|
|---|
| 98 | else:
|
|---|
| 99 | #Add --gen-suppressions=all to get suppression lines
|
|---|
| 100 | fid.write('LD_PRELOAD=%s \\\n' % self.valgrindlib)
|
|---|
| 101 | if IssmConfig('_HAVE_MPI_')[0]:
|
|---|
| 102 | fid.write('mpiexec -np %i %s --leak-check=full --suppressions=%s %s/%s %s %s/%s %s 2> %s.errlog >%s.outlog ' % \
|
|---|
| 103 | (self.np,self.valgrind,self.valgrindsup,self.codepath,executable,EnumToString(solution)[0],self.executionpath,dirname,modelname,modelname,modelname))
|
|---|
| 104 | else:
|
|---|
| 105 | fid.write('%s --leak-check=full --suppressions=%s %s/%s %s %s/%s %s 2> %s.errlog >%s.outlog ' % \
|
|---|
| 106 | (self.valgrind,self.valgrindsup,self.codepath,executable,EnumToString(solution)[0],self.executionpath,dirname,modelname,modelname,modelname))
|
|---|
| 107 |
|
|---|
| 108 | if not io_gather: #concatenate the output files:
|
|---|
| 109 | fid.write('\ncat %s.outbin.* > %s.outbin' % (modelname,modelname))
|
|---|
| 110 | fid.close()
|
|---|
| 111 |
|
|---|
| 112 | else: # Windows
|
|---|
| 113 |
|
|---|
| 114 | fid=open(modelname+'.bat','w')
|
|---|
| 115 | fid.write('@echo off\n')
|
|---|
| 116 | if self.interactive:
|
|---|
| 117 | fid.write('"%s/%s" %s "%s/%s" %s ' % (self.codepath,executable,EnumToString(solution)[0],self.executionpath,dirname,modelname))
|
|---|
| 118 | else:
|
|---|
| 119 | fid.write('"%s/%s" %s "%s/%s" %s 2> %s.errlog >%s.outlog' % \
|
|---|
| 120 | (self.codepath,executable,EnumToString(solution)[0],self.executionpath,dirname,modelname,modelname,modelname))
|
|---|
| 121 | fid.close()
|
|---|
| 122 |
|
|---|
| 123 | #in interactive mode, create a run file, and errlog and outlog file
|
|---|
| 124 | if self.interactive:
|
|---|
| 125 | fid=open(modelname+'.errlog','w')
|
|---|
| 126 | fid.close()
|
|---|
| 127 | fid=open(modelname+'.outlog','w')
|
|---|
| 128 | fid.close()
|
|---|
| 129 | # }}}
|
|---|
| 130 | def BuildKrigingQueueScript(self,modelname,solution,io_gather,isvalgrind,isgprof): # {{{
|
|---|
| 131 |
|
|---|
| 132 | #write queuing script
|
|---|
| 133 | if not m.ispc():
|
|---|
| 134 |
|
|---|
| 135 | fid=open(modelname+'.queue','w')
|
|---|
| 136 | fid.write('#!/bin/sh\n')
|
|---|
| 137 | if not isvalgrind:
|
|---|
| 138 | if self.interactive:
|
|---|
| 139 | fid.write('mpiexec -np %i %s/kriging.exe %s/%s %s ' % (self.np,self.codepath,self.executionpath,modelname,modelname))
|
|---|
| 140 | else:
|
|---|
| 141 | fid.write('mpiexec -np %i %s/kriging.exe %s/%s %s 2> %s.errlog >%s.outlog ' % (self.np,self.codepath,self.executionpath,modelname,modelname,modelname,modelname))
|
|---|
| 142 | elif isgprof:
|
|---|
| 143 | fid.write('\n gprof %s/kriging.exe gmon.out > %s.performance' & (self.codepath,modelname))
|
|---|
| 144 | else:
|
|---|
| 145 | #Add --gen-suppressions=all to get suppression lines
|
|---|
| 146 | fid.write('LD_PRELOAD=%s \\\n' % self.valgrindlib)
|
|---|
| 147 | fid.write('mpiexec -np %i %s --leak-check=full --suppressions=%s %s/kriging.exe %s/%s %s 2> %s.errlog >%s.outlog ' % \
|
|---|
| 148 | (self.np,self.valgrind,self.valgrindsup,self.codepath,self.executionpath,modelname,modelname,modelname,modelname))
|
|---|
| 149 | if not io_gather: #concatenate the output files:
|
|---|
| 150 | fid.write('\ncat %s.outbin.* > %s.outbin' % (modelname,modelname))
|
|---|
| 151 | fid.close()
|
|---|
| 152 |
|
|---|
| 153 | else: # Windows
|
|---|
| 154 |
|
|---|
| 155 | fid=open(modelname+'.bat','w')
|
|---|
| 156 | fid.write('@echo off\n')
|
|---|
| 157 | if self.interactive:
|
|---|
| 158 | fid.write('"%s/issm.exe" %s "%s/%s" %s ' % (self.codepath,EnumToString(solution)[0],self.executionpath,modelname,modelname))
|
|---|
| 159 | else:
|
|---|
| 160 | fid.write('"%s/issm.exe" %s "%s/%s" %s 2> %s.errlog >%s.outlog' % \
|
|---|
| 161 | (self.codepath,EnumToString(solution)[0],self.executionpath,modelname,modelname,modelname,modelname))
|
|---|
| 162 | fid.close()
|
|---|
| 163 |
|
|---|
| 164 | #in interactive mode, create a run file, and errlog and outlog file
|
|---|
| 165 | if self.interactive:
|
|---|
| 166 | fid=open(modelname+'.errlog','w')
|
|---|
| 167 | fid.close()
|
|---|
| 168 | fid=open(modelname+'.outlog','w')
|
|---|
| 169 | fid.close()
|
|---|
| 170 | # }}}
|
|---|
| 171 | def UploadQueueJob(self,modelname,dirname,filelist): # {{{
|
|---|
| 172 |
|
|---|
| 173 | #compress the files into one zip.
|
|---|
| 174 | compressstring='tar -zcf %s.tar.gz ' % dirname
|
|---|
| 175 | for file in filelist:
|
|---|
| 176 | compressstring += ' %s' % file
|
|---|
| 177 | if self.interactive:
|
|---|
| 178 | compressstring += ' %s.errlog %s.outlog ' % (modelname,modelname)
|
|---|
| 179 | subprocess.call(compressstring,shell=True)
|
|---|
| 180 |
|
|---|
| 181 | print('uploading input file and queueing script')
|
|---|
| 182 | issmscpout(self.name,self.executionpath,self.login,self.port,[dirname+'.tar.gz'])
|
|---|
| 183 |
|
|---|
| 184 | # }}}
|
|---|
| 185 | def LaunchQueueJob(self,modelname,dirname,filelist,restart): # {{{
|
|---|
| 186 |
|
|---|
| 187 | print('launching solution sequence on remote cluster')
|
|---|
| 188 | if restart:
|
|---|
| 189 | launchcommand='cd %s && cd %s chmod 777 %s.queue && ./%s.queue' % (self.executionpath,dirname,modelname,modelname)
|
|---|
| 190 | else:
|
|---|
| 191 | launchcommand='cd %s && rm -rf ./%s && mkdir %s && cd %s && mv ../%s.tar.gz ./ && tar -zxf %s.tar.gz && chmod 777 %s.queue && ./%s.queue' % \
|
|---|
| 192 | (self.executionpath,dirname,dirname,dirname,dirname,dirname,modelname,modelname)
|
|---|
| 193 | issmssh(self.name,self.login,self.port,launchcommand)
|
|---|
| 194 | # }}}
|
|---|
| 195 | def Download(self,dirname,filelist): # {{{
|
|---|
| 196 |
|
|---|
| 197 | if m.ispc():
|
|---|
| 198 | #do nothing
|
|---|
| 199 | return
|
|---|
| 200 |
|
|---|
| 201 | #copy files from cluster to current directory
|
|---|
| 202 | directory='%s/%s/' % (self.executionpath,dirname)
|
|---|
| 203 | issmscpin(self.name,self.login,self.port,directory,filelist)
|
|---|
| 204 | # }}}
|
|---|
![(please configure the [header_logo] section in trac.ini)](/trac/issm/chrome/common/trac_banner.png)
