Index: /issm/trunk-jpl/etc/environment.sh =================================================================== --- /issm/trunk-jpl/etc/environment.sh (revision 27335) +++ /issm/trunk-jpl/etc/environment.sh (revision 27336) @@ -381,4 +381,5 @@ if [ -d "${PETSC_ROOT}" ]; then export PETSC_ROOT # Used in installation of Gmsh + cpath_prepend "${PETSC_ROOT}/include" library_path_prepend "${PETSC_ROOT}/lib" ld_library_path_prepend "${PETSC_ROOT}/lib" @@ -519,4 +520,5 @@ SQLITE_ROOT="${ISSM_EXT_DIR}/sqlite/install" if [ -d "${SQLITE_ROOT}" ]; then + export SQLITE_ROOT # Used in installation of GDAL path_prepend "${SQLITE_ROOT}/bin" cpath_prepend "${SQLITE_ROOT}/include" Index: /issm/trunk-jpl/jenkins/eis-daskhub-python-nodules =================================================================== --- /issm/trunk-jpl/jenkins/eis-daskhub-python-nodules (revision 27336) +++ /issm/trunk-jpl/jenkins/eis-daskhub-python-nodules (revision 27336) @@ -0,0 +1,83 @@ +# NOTE: This configuration adds solid earth and Dakota capabilities to the +# basic build. + +#--------------------# +# ISSM Configuration # +#--------------------# + +ISSM_CONFIG='\ + --prefix="${ISSM_DIR}" \ + --disable-static \ + --enable-development \ + --enable-debugging \ + --with-numthreads=4 \ + --with-python-dir="/srv/conda/envs/notebook" \ + --with-python-version="3.9" \ + --with-python-numpy-dir="/srv/conda/envs/notebook/lib/python3.9/site-packages/numpy/core/include/numpy" \ + --with-fortran-lib="-L/usr/lib/x86_64-linux-gnu -lgfortran" \ + --with-mpi-include="${ISSM_EXT_DIR}/petsc/install/include" \ + --with-mpi-libflags="-L${ISSM_EXT_DIR}/petsc/install/lib -lmpi -lmpicxx -lmpifort" \ + --with-blas-lapack-dir="${ISSM_EXT_DIR}/petsc/install" \ + --with-metis-dir="${ISSM_EXT_DIR}/petsc/install" \ + --with-parmetis-dir="${ISSM_EXT_DIR}/petsc/install" \ + --with-scalapack-dir="${ISSM_EXT_DIR}/petsc/install" \ + --with-mumps-dir="${ISSM_EXT_DIR}/petsc/install" \ + --with-hdf5-dir="${ISSM_EXT_DIR}/petsc/install" \ + --with-petsc-dir="${ISSM_EXT_DIR}/petsc/install" \ + --with-gsl-dir="${ISSM_EXT_DIR}/gsl/install" \ + --with-boost-dir="${ISSM_EXT_DIR}/boost/install" \ + --with-dakota-dir="${ISSM_EXT_DIR}/dakota/install" \ + --with-proj-dir="${ISSM_EXT_DIR}/proj/install" \ + --with-triangle-dir="${ISSM_EXT_DIR}/triangle/install" \ + --with-chaco-dir="${ISSM_EXT_DIR}/chaco/install" \ + --with-m1qn3-dir="${ISSM_EXT_DIR}/m1qn3/install" \ + --with-semic-dir="${ISSM_EXT_DIR}/semic/install" \ +' + +#-------------------# +# External Packages # +#-------------------# + +EXTERNALPACKAGES=" + autotools install-linux.sh + cmake install.sh + petsc install-3.16-linux.sh + gsl install.sh + boost install-1.7-linux.sh + dakota install-6.2-linux.sh + curl install-7-linux.sh + netcdf install-4.7-parallel.sh + proj install-6.sh + gdal install-3-python.sh + gshhg install.sh + gmt install-6-linux.sh + gmsh install-4-linux.sh + triangle install-linux.sh + chaco install.sh + m1qn3 install.sh + semic install.sh +" + +#---------# +# Testing # +#---------# + +# Test suites +MATLAB_TEST=0 +PYTHON_TEST=0 +JAVASCRIPT_TEST=0 +EXAMPLES_TEST=0 + +# Number of CPUs used in ISSM compilation +# +# NOTE: One is usually safer as some packages are very sensitive to parallel +# compilation +# +NUMCPUS_INSTALL=8 + +# Number of CPUs used in the nightly runs +NUMCPUS_RUN=1 + +# Nightly run options +MATLAB_NROPTIONS="" +PYTHON_NROPTIONS="" Index: /issm/trunk-jpl/jenkins/eis-smce-binaries =================================================================== --- /issm/trunk-jpl/jenkins/eis-smce-binaries (revision 27336) +++ /issm/trunk-jpl/jenkins/eis-smce-binaries (revision 27336) @@ -0,0 +1,82 @@ +# NOTE: This configuration adds solid earth and Dakota capabilities to the +# basic build. + +#--------------------# +# ISSM Configuration # +#--------------------# + +ISSM_CONFIG='\ + --prefix="${ISSM_DIR}" \ + --disable-static \ + --with-wrappers=no \ + --enable-development \ + --enable-debugging \ + --with-numthreads=48 \ + --with-fortran-lib="-L/usr/lib/gcc/x86_64-linux-gnu/9 -lgfortran" \ + --with-mpi-include="/opt/intel/mpi/2021.4.0/include" \ + --with-mpi-libflags="-L/opt/intel/mpi/2021.4.0/lib -lmpi -lmpicxx -lmpifort" \ + --with-blas-lapack-dir="${ISSM_EXT_DIR}/petsc/install" \ + --with-metis-dir="${ISSM_EXT_DIR}/petsc/install" \ + --with-parmetis-dir="${ISSM_EXT_DIR}/petsc/install" \ + --with-scalapack-dir="${ISSM_EXT_DIR}/petsc/install" \ + --with-mumps-dir="${ISSM_EXT_DIR}/petsc/install" \ + --with-hdf5-dir="${ISSM_EXT_DIR}/petsc/install" \ + --with-petsc-dir="${ISSM_EXT_DIR}/petsc/install" \ + --with-gsl-dir="${ISSM_EXT_DIR}/gsl/install" \ + --with-boost-dir="${ISSM_EXT_DIR}/boost/install" \ + --with-dakota-dir="${ISSM_EXT_DIR}/dakota/install" \ + --with-proj-dir="${ISSM_EXT_DIR}/proj/install" \ + --with-triangle-dir="${ISSM_EXT_DIR}/triangle/install" \ + --with-chaco-dir="${ISSM_EXT_DIR}/chaco/install" \ + --with-m1qn3-dir="${ISSM_EXT_DIR}/m1qn3/install" \ + --with-semic-dir="${ISSM_EXT_DIR}/semic/install" +' + +#-------------------# +# External Packages # +#-------------------# + +EXTERNALPACKAGES=" + autotools install-linux.sh + cmake install.sh + petsc install-3.12-linux.sh + gsl install.sh + boost install-1.7-linux.sh + dakota install-6.2-linux.sh + curl install-7-linux.sh + netcdf install-4.7-parallel.sh + sqlite install.sh + proj install-6.sh + gdal install-3-python.sh + gshhg install.sh + gmt install-6-linux.sh + gmsh install-4.sh + triangle install-linux.sh + chaco install.sh + m1qn3 install.sh + semic install.sh +" + +#---------# +# Testing # +#---------# + +# Test suites +MATLAB_TEST=0 +PYTHON_TEST=0 +JAVASCRIPT_TEST=0 +EXAMPLES_TEST=0 + +# Number of CPUs used in ISSM compilation +# +# NOTE: One is usually safer as some packages are very sensitive to parallel +# compilation +# +NUMCPUS_INSTALL=8 + +# Number of CPUs used in the nightly runs +NUMCPUS_RUN=1 + +# Nightly run options +MATLAB_NROPTIONS="" +PYTHON_NROPTIONS="" Index: /issm/trunk-jpl/src/m/classes/clusters/eis_nasa_smce.py =================================================================== --- /issm/trunk-jpl/src/m/classes/clusters/eis_nasa_smce.py (revision 27336) +++ /issm/trunk-jpl/src/m/classes/clusters/eis_nasa_smce.py (revision 27336) @@ -0,0 +1,217 @@ +import os +import shutil +import subprocess + +try: + from eis_nasa_smce_settings import eis_nasa_smce_settings +except ImportError: + print('You need eis_nasa_smce_settings.py to proceed, check presence and sys.path') +from fielddisplay import fielddisplay +from helpers import * +from IssmConfig import IssmConfig +from issmssh import issmssh +from MatlabFuncs import * +from pairoptions import pairoptions + +class eis_nasa_smce(object): + """EIS_NASA_SMCE cluster class definition + + Usage: + cluster = eis_nasa_smce() + cluster = eis_nasa_smce('np', 3) + cluster = eis_nasa_scme('np', 3, 'login', 'username') + """ + + def __init__(self, *args): # {{{ + self.name = '52.10.233.96' + self.login = 'jdquinn1' + self.idfile = '~/.ssh/eis-nasa-smce' + self.modules = ['intelmpi'] + self.numnodes = 4 + self.cpuspernode = 1 + self.port = 0 + self.time = 12 * 60 * 60 + self.processor = 'skylake' + self.partition = 'sealevel-c5xl-spot' + self.srcpath = '/efs/issm-new/binaries/repos/trunk-jpl-working' + self.extpkgpath = '/efs/issm-new/binaries/ext' + self.codepath = '/efs/issm-new/binaries/repos/trunk-jpl-working/bin' + self.executionpath = '~/issm-exec' + self.interactive = 0 + self.numstreams = 1 + self.hyperthreading = 0 + self.email = '' + + # Use provided options to change fields + options = pairoptions(*args) + + # Initialize cluster using user settings if provided + try: + self = eis_nasa_smce_settings(self) + except NameError: + print('eis_nasa_smce_settings.py not found, using default settings') + + # OK get other fields + self = options.AssignObjectFields(self) + # }}} + + def __repr__(self): # {{{ + # Display the object + s = 'class eis_nasa_smce object\n' + s += ' name: {}\n'.format(self.name) + s += ' login: {}\n'.format(self.login) + s += ' idfile: {}\n'.format(self.idfile) + s += ' modules: {}\n'.format(strjoin(self.modules, ', ')) + s += ' numnodes: {}\n'.format(self.numnodes) + s += ' cpuspernode: {}\n'.format(self.cpuspernode) + s += ' np: {}\n'.format(self.nprocs()) + s += ' port: {}\n'.format(self.port) + s += ' time: {}\n'.format(self.time) + s += ' processor: {}\n'.format(self.processor) + s += ' partition: {}\n'.format(self.partition) + s += ' srcpath: {}\n'.format(self.srcpath) + s += ' extpkgpath: {}\n'.format(self.extpkgpath) + s += ' codepath: {}\n'.format(self.codepath) + s += ' executionpath: {}\n'.format(self.executionpath) + s += ' interactive: {}\n'.format(self.interactive) + s += ' numstreams: {}\n'.format(self.numstreams) + s += ' hyperthreading: {}\n'.format(self.hyperthreading) + return s + # }}} + + def nprocs(self): # {{{ + return self.numnodes * self.cpuspernode + # }}} + + def checkconsistency(self, md, solution, analyses): # {{{ + # Now, check cluster.cpuspernode according to processor type + if self.processor == 'skylake': + if self.cpuspernode > 14 or self.cpuspernode < 1: + md = md.checkmessage('cpuspernode should be between 1 and 14 for \'skyw\' processors in hyperthreading mode') + else: + md = md.checkmessage('unknown processor type, should be \'skylake\'') + + # Miscellaneous + if not self.login: + md = md.checkmessage('login empty') + if self.port: + md = md.checkmessage('port must be set to 0 as we do not have an SSH tunnel') + if not self.codepath: + md = md.checkmessage('codepath empty') + if not self.executionpath: + md = md.checkmessage('executionpath empty') + + return self + # }}} + + def BuildQueueScript(self, dirname, modelname, solution, io_gather, isvalgrind, isgprof, isdakota, isoceancoupling): # {{{ + if isgprof: + print('gprof not supported by cluster, ignoring...') + + issmexec = 'issm.exe' + mpiexec = 'mpiexec' # Set to alternative mpiexec if desired + + if isdakota: + version = IssmConfig('_DAKOTA_VERSION_')[0:2] + version = float(str(version[0])) + if version >= 6: + issmexec = 'issm_dakota.exe' + if isoceancoupling: + issmexec = 'issm_ocean.exe' + + # Write queuing script + fid = open(modelname + '.queue', 'w') + + fid.write('#!/bin/bash\n') + fid.write('#SBATCH --partition={} \n'.format(self.partition)) + fid.write('#SBATCH -J {} \n'.format(modelname)) + fid.write('#SBATCH -o {}.outlog \n'.format(modelname)) + fid.write('#SBATCH -e {}.errlog \n'.format(modelname)) + fid.write('#SBATCH --nodes={} \n'.format(self.numnodes)) + fid.write('#SBATCH --ntasks-per-node={} \n'.format(self.cpuspernode)) + fid.write('#SBATCH --cpus-per-task={} \n'.format(self.numstreams)) + fid.write('#SBATCH -t {:02d}:{:02d}:00 \n'.format(int(floor(self.time / 3600)), int(floor(self.time % 3600) / 60))) + if (self.email.find('@')>-1): + fid.write('#SBATCH --mail-user={} \n'.format(self.email)) + fid.write('#SBATCH --mail-type=BEGIN,END,FAIL \n\n') + fid.write('source /etc/profile\n') + fid.write('source /shared/spack/share/spack/setup-env.sh\n') + for i in range(len(self.modules)): + fid.write('module load {} &> /dev/null\n'.format(self.modules[i])) + fid.write('export MPI_GROUP_MAX=64\n\n') + fid.write('export MPI_UNBUFFERED_STDIO=true\n\n') + fid.write('export PATH="$PATH:/opt/slurm/bin"\n') + fid.write('export PATH="$PATH:."\n\n') + fid.write('export ISSM_DIR="{}"\n'.format(self.srcpath)) + fid.write('export ISSM_EXT_DIR="{}"\n'.format(self.extpkgpath)) + fid.write('source $ISSM_DIR/etc/environment.sh\n') + fid.write('cd {}/{}/\n\n'.format(self.executionpath, dirname)) + fid.write('{} -n {} {}/{} {} {}/{} {}\n'.format(mpiexec, self.nprocs(), self.codepath, issmexec, solution, self.executionpath, dirname, modelname)) + + if not io_gather: # concatenate the output files + fid.write('cat {}.outbin.* > {}.outbin'.format(modelname, modelname)) + fid.close() + + # In interactive mode, create a run file, and errlog and outlog file + if self.interactive: + fid = open(modelname + '.run', 'w') + if not isvalgrind: + fid.write('{} -np {} {}/{} {} {}/{} {}\n'.format(mpiexec, self.nprocs(), self.codepath, issmexec, solution, self.executionpath, dirname, modelname)) + else: + fid.write('{} -np {} valgrind --leak-check=full {}/{} {} {}/{} {}\n'.format(mpiexec, self.nprocs(), self.codepath, issmexec, solution, self.executionpath, dirname, modelname)) + if not io_gather: # concatenate the output files + fid.write('cat {}.outbin.* > {}.outbin'.format(modelname, modelname)) + fid.close() + fid = open(modelname + '.errlog', 'w') # TODO: Change this to system call (touch )? + fid.close() + fid = open(modelname + '.outlog', 'w') # TODO: Change this to system call (touch )? + fid.close() + # }}} + + def UploadQueueJob(self, modelname, dirname, filelist): # {{{ + # Compress the files into one zip + compressstring = 'tar -zcf {}.tar.gz'.format(dirname) + for file in filelist: + compressstring += ' {}'.format(file) + if self.interactive: + compressstring += ' {}.run {}.errlog {}.outlog'.format(modelname, modelname, modelname) + subprocess.call(compressstring, shell=True) + + print('uploading input file and queueing script') + if self.interactive: + directory = '{}/Interactive{}'.format(self.executionpath, self.interactive) + else: + directory = self.executionpath + + # NOTE: Replacement for issmscpout(self.name, directory, self.login, self.port, ['{}.tar.gz'.format(dirname)]) + copystring = 'cp {}.tar.gz /efs/issm/tmp'.format(dirname) + subprocess.call(copystring, shell=True) + # }}} + + def LaunchQueueJob(self, modelname, dirname, filelist, restart, batch): # {{{ + if self.interactive: + if not isempty(restart): + launchcommand = 'cd {}/Interactive{}'.format(self.executionpath, self.interactive) + else: + launchcommand = 'cd {}/Interactive{} && tar -zxf {}.tar.gz'.format(self.executionpath, self.interactive, dirname) + else: + if not isempty(restart): + launchcommand = 'cd {} && cd {} && sbatch {}.queue'.format(self.executionpath, dirname, modelname) + else: + launchcommand = 'cd {} && rm -rf {} && mkdir {} && cd {} && cp /efs/issm/tmp/{}.tar.gz . && tar -zxf {}.tar.gz && /opt/slurm/bin/sbatch {}.queue'.format(self.executionpath, dirname, dirname, dirname, dirname, dirname, modelname) + + print('launching solution sequence on remote cluster') + + # NOTE: Replacement for issmssh(self.name, self.login, self.port, launchcommand) + subprocess.call('ssh -l {} -i {} {} "{}"'.format(self.login, self.idfile, self.name, launchcommand), shell=True) + # }}} + + def Download(self, dirname, filelist): # {{{ + # Copy files from cluster to current directory + + # NOTE: Replacement for issmscpin(self.name, self.login, self.port, directory, filelist) + directory = '{}/{}/'.format(self.executionpath, dirname) + fileliststr = '{' + ','.join([str(x) for x in filelist]) + '}' + downloadcommand = 'scp -i {} {}@{}:{} {}/.'.format(self.idfile, self.login, self.name, os.path.join(directory, fileliststr), os.getcwd()) + subprocess.call(downloadcommand, shell=True) + # }}} Index: sm/trunk-jpl/src/m/classes/clusters/smce_eis.py =================================================================== --- /issm/trunk-jpl/src/m/classes/clusters/smce_eis.py (revision 27335) +++ (revision ) @@ -1,216 +1,0 @@ -import os -import shutil -import subprocess - -try: - from smce_eis_settings import smce_eis_settings -except ImportError: - print('You need smce_eis_settings.py to proceed, check presence and sys.path') -from fielddisplay import fielddisplay -from helpers import * -from IssmConfig import IssmConfig -from issmssh import issmssh -from MatlabFuncs import * -from pairoptions import pairoptions - -class smce_eis(object): - """SMCE_EIS cluster class definition - - Usage: - cluster = smce_eis() - cluster = smce_eis('np', 3) - cluster = smce_eis('np', 3, 'login', 'username') - """ - - def __init__(self, *args): # {{{ - self.name = '52.54.75.45' - self.login = 'jdquinn1' - self.idfile = '~/.ssh/smce-aws-eis-pilot' - self.modules = [] - self.numnodes = 8 - self.cpuspernode = 1 - self.port = 0 - self.time = 12 * 60 * 60 - self.processor = 'skylake' - self.srcpath = '/shared/issm/issm/trunk-jpl' - self.codepath = '/shared/issm/issm/trunk-jpl/bin' - self.executionpath = '/home/jdquinn1/issm-exec' - self.interactive = 0 - self.numstreams = 1 - self.hyperthreading = 0 - self.email = '' - - # Use provided options to change fields - options = pairoptions(*args) - - # Initialize cluster using user settings if provided - try: - self = smce_eis_settings(self) - except NameError: - print('smce_eis_settings.py not found, using default settings') - - # OK get other fields - self = options.AssignObjectFields(self) - # }}} - - def __repr__(self): # {{{ - # Display the object - s = 'class smce_eis object\n' - s += ' name: {}\n'.format(self.name) - s += ' login: {}\n'.format(self.login) - s += ' idfile: {}\n'.format(self.idfile) - s += ' modules: {}\n'.format(strjoin(self.modules, ', ')) - s += ' numnodes: {}\n'.format(self.numnodes) - s += ' cpuspernode: {}\n'.format(self.cpuspernode) - s += ' np: {}\n'.format(self.nprocs()) - s += ' port: {}\n'.format(self.port) - s += ' time: {}\n'.format(self.time) - s += ' processor: {}\n'.format(self.processor) - s += ' srcpath: {}\n'.format(self.srcpath) - s += ' codepath: {}\n'.format(self.codepath) - s += ' executionpath: {}\n'.format(self.executionpath) - s += ' interactive: {}\n'.format(self.interactive) - s += ' numstreams: {}\n'.format(self.numstreams) - s += ' hyperthreading: {}\n'.format(self.hyperthreading) - return s - # }}} - - def nprocs(self): # {{{ - return self.numnodes * self.cpuspernode - # }}} - - def checkconsistency(self, md, solution, analyses): # {{{ - # Now, check cluster.cpuspernode according to processor type - if self.processor == 'skylake': - if self.cpuspernode > 14 or self.cpuspernode < 1: - md = md.checkmessage('cpuspernode should be between 1 and 14 for \'skyw\' processors in hyperthreading mode') - else: - md = md.checkmessage('unknown processor type, should be \'skylake\'') - - # Miscellaneous - if not self.login: - md = md.checkmessage('login empty') - if self.port: - md = md.checkmessage('port must be set to 0 as we do not have an SSH tunnel') - if not self.codepath: - md = md.checkmessage('codepath empty') - if not self.executionpath: - md = md.checkmessage('executionpath empty') - - return self - # }}} - - def BuildQueueScript(self, dirname, modelname, solution, io_gather, isvalgrind, isgprof, isdakota, isoceancoupling): # {{{ - if isgprof: - print('gprof not supported by cluster, ignoring...') - - executable = 'issm.exe' - if isdakota: - version = IssmConfig('_DAKOTA_VERSION_')[0:2] - version = float(str(version[0])) - if version >= 6: - executable = 'issm_dakota.exe' - if isoceancoupling: - executable = 'issm_ocean.exe' - - # Write queuing script - fid = open(modelname + '.queue', 'w') - - fid.write('#!/bin/bash\n') - fid.write('#SBATCH --partition=hpc-spot \n') - fid.write('#SBATCH -J {} \n'.format(modelname)) - fid.write('#SBATCH -o {}.outlog \n'.format(modelname)) - fid.write('#SBATCH -e {}.errlog \n'.format(modelname)) - fid.write('#SBATCH --nodes={} \n'.format(self.numnodes)) - fid.write('#SBATCH --ntasks-per-node={} \n'.format(self.cpuspernode)) - fid.write('#SBATCH --cpus-per-task={} \n'.format(self.numstreams)) - fid.write('#SBATCH -t {:02d}:{:02d}:00 \n'.format(int(floor(self.time / 3600)), int(floor(self.time % 3600) / 60))) - if (self.email.find('@')>-1): - fid.write('#SBATCH --mail-user={} \n'.format(self.email)) - fid.write('#SBATCH --mail-type=BEGIN,END,FAIL \n\n') - # for i in range(len(self.modules)): - # fid.write('module load {}\n'.format(self.modules[i])) - fid.write('export MPI_GROUP_MAX=64\n\n') - fid.write('export MPI_UNBUFFERED_STDIO=true\n\n') - fid.write('export PATH="$PATH:/opt/slurm/bin"\n') # TODO: Figure out how to add this to PATH by sourcing environment script - fid.write('export PATH="$PATH:."\n\n') - fid.write('export ISSM_DIR="{}"\n'.format(self.srcpath)) # FIXME - fid.write('export HYDRA_HOST_FILE={}/{}/'.format(self.executionpath, dirname)) - fid.write('${USER}-hydranodes-sj:${SLURM_JOB_ID}\n') - #fid.write('source /usr/share/modules/init/bash\n') - #fid.write('source /efs/spack-justin/share/spack/setup-env.sh\n') - #fid.write('module load gcc-9.3.0-gcc-9.3.0-ug2hqa3\n') - fid.write('touch ${HYDRA_HOST_FILE}\n') - fid.write('/efs/mrilee/snodelist/build/snodelist -m -f "%h%[:]c" >> ${HYDRA_HOST_FILE}\n') - fid.write('source $ISSM_DIR/etc/environment.sh\n') # FIXME - fid.write('cd {}/{}/\n\n'.format(self.executionpath, dirname)) - - fid.write('mpiexec -np {} {}/{} {} {}/{} {}\n'.format(self.nprocs(), self.codepath, executable, solution, self.executionpath, dirname, modelname)) - - if not io_gather: # concatenate the output files - fid.write('cat {}.outbin.* > {}.outbin'.format(modelname, modelname)) - fid.close() - - # In interactive mode, create a run file, and errlog and outlog file - if self.interactive: - fid = open(modelname + '.run', 'w') - if not isvalgrind: - fid.write('mpiexec -np {} {}/{} {} {}/{} {}\n'.format(self.nprocs(), self.codepath, executable, solution, self.executionpath, dirname, modelname)) - else: - fid.write('mpiexec -np {} valgrind --leak-check=full {}/{} {} {}/{} {}\n'.format(self.nprocs(), self.codepath, executable, solution, self.executionpath, dirname, modelname)) - if not io_gather: # concatenate the output files - fid.write('cat {}.outbin.* > {}.outbin'.format(modelname, modelname)) - fid.close() - fid = open(modelname + '.errlog', 'w') # TODO: Change this to system call (touch )? - fid.close() - fid = open(modelname + '.outlog', 'w') # TODO: Change this to system call (touch )? - fid.close() - # }}} - - def UploadQueueJob(self, modelname, dirname, filelist): # {{{ - # Compress the files into one zip - compressstring = 'tar -zcf {}.tar.gz'.format(dirname) - for file in filelist: - compressstring += ' {}'.format(file) - if self.interactive: - compressstring += ' {}.run {}.errlog {}.outlog'.format(modelname, modelname, modelname) - subprocess.call(compressstring, shell=True) - - print('uploading input file and queuing script') - if self.interactive: - directory = '{}/Interactive{}'.format(self.executionpath, self.interactive) - else: - directory = self.executionpath - - # NOTE: Replacement for issmscpout(self.name, directory, self.login, self.port, ['{}.tar.gz'.format(dirname)]) - copystring = 'cp {}.tar.gz /efs/issm-tmp/'.format(dirname, dirname) - subprocess.call(copystring, shell=True) - # }}} - - def LaunchQueueJob(self, modelname, dirname, filelist, restart, batch): # {{{ - if self.interactive: - if not isempty(restart): - launchcommand = 'cd {}/Interactive{}'.format(self.executionpath, self.interactive) - else: - launchcommand = 'cd {}/Interactive{} && tar -zxf {}.tar.gz'.format(self.executionpath, self.interactive, dirname) - else: - if not isempty(restart): - launchcommand = 'cd {} && cd {} && /opt/slurm/bin/sbatch {}.queue'.format(self.executionpath, dirname, modelname) - else: - launchcommand = 'cd {} && rm -rf ./{} && mkdir {} && cd {} && cp /efs/issm-tmp/{}.tar.gz ./ && tar -zxf {}.tar.gz && /opt/slurm/bin/sbatch {}.queue'.format(self.executionpath, dirname, dirname, dirname, dirname, dirname, modelname) - - print('launching solution sequence on remote cluster') - - # NOTE: Replacement for issmssh(self.name, self.login, self.port, launchcommand) - subprocess.call('ssh -l {} -i {} {} "{}"'.format(self.login, self.idfile, self.name, launchcommand), shell=True) - # }}} - - def Download(self, dirname, filelist): # {{{ - # Copy files from cluster to current directory - - # NOTE: Replacement for issmscpin(self.name, self.login, self.port, directory, filelist) - directory = '{}/{}/'.format(self.executionpath, dirname) - fileliststr = "'{" + ','.join([str(x) for x in filelist]) + "}'" - downloadcommand = 'scp -T -i {} {}@{}:{} {}/.'.format(self.idfile, self.login, self.name, os.path.join(directory, fileliststr), os.getcwd()) - subprocess.call(downloadcommand, shell=True) - # }}} Index: /issm/trunk-jpl/test/NightlyRun/test101eisnasasmce.py =================================================================== --- /issm/trunk-jpl/test/NightlyRun/test101eisnasasmce.py (revision 27336) +++ /issm/trunk-jpl/test/NightlyRun/test101eisnasasmce.py (revision 27336) @@ -0,0 +1,55 @@ +#Test Name: SquareShelfConstrainedStressSSA2d +from model import * +from socket import gethostname +from triangle import triangle +from setmask import setmask +from parameterize import parameterize +from setflowequation import setflowequation +from solve import solve +from massfluxatgate import massfluxatgate +from generic import generic +from eis_nasa_smce import eis_nasa_smce + +md = triangle(model(), '../Exp/Square.exp', 50000) +md = setmask(md, 'all', '') +md = parameterize(md, '../Par/SquareShelfConstrained.py') +md = setflowequation(md, 'SSA', 'all') +md.cluster = generic('name', gethostname(), 'np', 2) + +if True: + cluster = eis_nasa_smce() + cluster.partition = 'sealevel-c5xl-spot' + md.cluster = cluster + +#outputs +md.stressbalance.requested_outputs = ['default', 'DeviatoricStressxx', 'DeviatoricStressyy', 'DeviatoricStressxy', 'MassFlux1', 'MassFlux2', 'MassFlux3', 'MassFlux4', 'MassFlux5', 'MassFlux6'] +md.outputdefinition.definitions = [massfluxatgate('name', 'MassFlux1', 'profilename', '../Exp/MassFlux1.exp', 'definitionstring', 'Outputdefinition1'), + massfluxatgate('name', 'MassFlux2', 'profilename', '../Exp/MassFlux2.exp', 'definitionstring', 'Outputdefinition2'), + massfluxatgate('name', 'MassFlux3', 'profilename', '../Exp/MassFlux3.exp', 'definitionstring', 'Outputdefinition3'), + massfluxatgate('name', 'MassFlux4', 'profilename', '../Exp/MassFlux4.exp', 'definitionstring', 'Outputdefinition4'), + massfluxatgate('name', 'MassFlux5', 'profilename', '../Exp/MassFlux5.exp', 'definitionstring', 'Outputdefinition5'), + massfluxatgate('name', 'MassFlux6', 'profilename', '../Exp/MassFlux6.exp', 'definitionstring', 'Outputdefinition6')] + +md = solve(md, 'Stressbalance') + +#Fields and tolerances to track changes +field_names = ['Vx', 'Vy', 'Vel', 'Pressure', + 'DeviatoricStressxx', 'DeviatoricStressyy', 'DeviatoricStressxy', + 'MassFlux1', 'MassFlux2', 'MassFlux3', 'MassFlux4', 'MassFlux5', 'MassFlux6'] +field_tolerances = [3e-13, 1e-13, 1e-13, 1e-13, + 2e-13, 1e-13, 2e-13, + 1e-13, 1e-13, 1e-13, + 1e-13, 1e-13, 1e-13] +field_values = [md.results.StressbalanceSolution.Vx, + md.results.StressbalanceSolution.Vy, + md.results.StressbalanceSolution.Vel, + md.results.StressbalanceSolution.Pressure, + md.results.StressbalanceSolution.DeviatoricStressxx, + md.results.StressbalanceSolution.DeviatoricStressyy, + md.results.StressbalanceSolution.DeviatoricStressxy, + md.results.StressbalanceSolution.MassFlux1, + md.results.StressbalanceSolution.MassFlux2, + md.results.StressbalanceSolution.MassFlux3, + md.results.StressbalanceSolution.MassFlux4, + md.results.StressbalanceSolution.MassFlux5, + md.results.StressbalanceSolution.MassFlux6]