Index: /issm/branches/trunk-larour-SLPS2022/src/m/classes/clusters/pfe.m
===================================================================
--- /issm/branches/trunk-larour-SLPS2022/src/m/classes/clusters/pfe.m	(revision 27223)
+++ /issm/branches/trunk-larour-SLPS2022/src/m/classes/clusters/pfe.m	(revision 27224)
@@ -181,5 +181,5 @@
 			fprintf(fid,'cd %s/%s/\n\n',cluster.executionpath,dirname);
 			if ~isvalgrind,
-				fprintf(fid,'/u/scicon/tools/bin/toss3/several_tries mpiexec -np %i mbind.x -cs -n%i %s/%s %s %s/%s %s\n',cluster.nprocs(),cluster.cpuspernode,cluster.codepath,executable,solution,cluster.executionpath,dirname,modelname);
+				fprintf(fid,'/u/scicon/tools/bin/toss3/several_tries mpiexec -np %i /u/scicon/tools/bin/mbind.x -cs -n%i %s/%s %s %s/%s %s\n',cluster.nprocs(),cluster.cpuspernode,cluster.codepath,executable,solution,cluster.executionpath,dirname,modelname);
 			else
 				fprintf(fid,'mpiexec -np %i valgrind --leak-check=full %s/%s %s %s %s\n',cluster.nprocs(),cluster.codepath,executable,solution,[cluster.executionpath '/' dirname],modelname);
Index: /issm/branches/trunk-larour-SLPS2022/src/m/classes/clusters/pfeuq.m
===================================================================
--- /issm/branches/trunk-larour-SLPS2022/src/m/classes/clusters/pfeuq.m	(revision 27224)
+++ /issm/branches/trunk-larour-SLPS2022/src/m/classes/clusters/pfeuq.m	(revision 27224)
@@ -0,0 +1,268 @@
+%PFEUQ cluster class definition
+%
+%   Usage:
+%      cluster=pfeuq();
+%      cluster=pfeuq('np',3);
+%      cluster=pfeuq('np',3,'login','username');
+
+classdef pfeuq
+	properties (SetAccess=public)
+		% {{{
+		name           = 'pfe'
+		login          = '';
+		modules        = {'comp-intel/2018.3.222' 'mpi-intel/2018.3.222' 'scicon/app-tools'};
+		concurrent_evaluations=3;
+		numnodes_per_evaluation = 6;
+		cpuspernode    = 27;
+		port           = 1025;
+		queue          = 'long';
+		time           = 12*60;
+		processor      = 'bro';
+		srcpath        = '';
+		codepath       = '';
+		executionpath  = '';
+		grouplist      = '';
+		interactive    = 0;
+		bbftp          = 0;
+		numstreams     = 8;
+		hyperthreading = 0;
+	end
+	%}}}
+	methods
+		function cluster=pfe(varargin) % {{{
+
+			%initialize cluster using default settings if provided
+			if (exist('pfe_settings')==2), pfe_settings; end
+
+			%use provided options to change fields
+			cluster=AssignObjectFields(pairoptions(varargin{:}),cluster);
+		end
+		%}}}
+		function disp(cluster) % {{{
+			% display the object
+			disp(sprintf('class ''%s'' object ''%s'' = ',class(cluster),inputname(1)));
+			disp(sprintf('    name: %s',cluster.name));
+			disp(sprintf('    login: %s',cluster.login));
+			disp(sprintf('    modules: %s',strjoin(cluster.modules,', ')));
+			disp(sprintf('    concurrent_evaluations: %i',cluster.concurrent_evaluations));
+			disp(sprintf('    numnodes_per_evaluation: %i',cluster.numnodes_per_evaluation));
+			disp(sprintf('    cpuspernode: %i',cluster.cpuspernode));
+			disp(sprintf('    port: %i',cluster.port));
+			disp(sprintf('    queue: %s',cluster.queue));
+			disp(sprintf('    time: %i',cluster.time));
+			disp(sprintf('    processor: %s',cluster.processor));
+			disp(sprintf('    srcpath: %s',cluster.srcpath));
+			disp(sprintf('    codepath: %s',cluster.codepath));
+			disp(sprintf('    executionpath: %s',cluster.executionpath));
+			disp(sprintf('    grouplist: %s',cluster.grouplist));
+			disp(sprintf('    interactive: %i',cluster.interactive));
+			disp(sprintf('    bbftp: %s',cluster.bbftp));
+			disp(sprintf('    numstreams: %s',cluster.numstreams));
+			disp(sprintf('    hyperthreading: %s',cluster.hyperthreading));
+		end
+		%}}}
+		function md = checkconsistency(cluster,md,solution,analyses) % {{{
+
+			%now, check cluster.cpuspernode according to processor type
+			if strcmpi(cluster.processor,'ivy'),
+				if cluster.hyperthreading,
+					if ((cluster.cpuspernode>40 ) | (cluster.cpuspernode<1)),
+						md = checkmessage(md,'cpuspernode should be between 1 and 40 for ''ivy'' processors in hyperthreading mode');
+					end
+				else
+					if ((cluster.cpuspernode>20 ) | (cluster.cpuspernode<1)),
+						md = checkmessage(md,'cpuspernode should be between 1 and 20 for ''ivy'' processors');
+					end
+				end
+			elseif strcmpi(cluster.processor,'bro'),
+				if cluster.hyperthreading,
+					if ((cluster.cpuspernode>56 ) | (cluster.cpuspernode<1)),
+						md = checkmessage(md,'cpuspernode should be between 1 and 56 for ''bro'' processors in hyperthreading mode');
+					end
+				else
+					if ((cluster.cpuspernode>28 ) | (cluster.cpuspernode<1)),
+						md = checkmessage(md,'cpuspernode should be between 1 and 28 for ''bro'' processors');
+					end
+				end
+			elseif strcmpi(cluster.processor,'has'),
+				if cluster.hyperthreading,
+					if ((cluster.cpuspernode>48 ) | (cluster.cpuspernode<1)),
+						md = checkmessage(md,'cpuspernode should be between 1 and 48 for ''has'' processors in hyperthreading mode');
+					end
+				else
+					if ((cluster.cpuspernode>24 ) | (cluster.cpuspernode<1)),
+						md = checkmessage(md,'cpuspernode should be between 1 and 24 for ''has'' processors');
+					end
+				end
+			
+			elseif strcmpi(cluster.processor,'san'),
+				if cluster.hyperthreading,
+					if ((cluster.cpuspernode>32 ) | (cluster.cpuspernode<1)),
+						md = checkmessage(md,'cpuspernode should be between 1 and 32 for ''san'' processors in hyperthreading mode');
+					end
+				else
+					if ((cluster.cpuspernode>16 ) | (cluster.cpuspernode<1)),
+						md = checkmessage(md,'cpuspernode should be between 1 and 16 for ''san'' processors');
+					end
+				end
+
+			elseif strcmpi(cluster.processor,'cas_ait'),
+				if cluster.hyperthreading,
+					if ((cluster.cpuspernode>80 ) | (cluster.cpuspernode<1)),
+						md = checkmessage(md,'cpuspernode should be between 1 and 80 for ''cas_ait'' processors in hyperthreading mode');
+					end
+				else
+					if ((cluster.cpuspernode>40 ) | (cluster.cpuspernode<1)),
+						md = checkmessage(md,'cpuspernode should be between 1 and 40 for ''cas_ait'' processors');
+					end
+				end
+			
+			else
+				md = checkmessage(md,'unknown processor type, should be ''bro'', ''has'', ''ivy'', ''san'', or ''cas_ait''');
+			end
+
+			%Miscellaneous
+			if isempty(cluster.login), md = checkmessage(md,'login empty'); end
+			if isempty(cluster.srcpath), md = checkmessage(md,'srcpath empty'); end
+			if isempty(cluster.codepath), md = checkmessage(md,'codepath empty'); end
+			if isempty(cluster.executionpath), md = checkmessage(md,'executionpath empty'); end
+			if isempty(cluster.grouplist), md = checkmessage(md,'grouplist empty'); end
+
+		end
+		%}}}
+		function numprocs=nprocs(cluster) % {{{
+			%compute number of processors
+			numprocs=cluster.concurrent_evaluations*cluster.numnodes_per_evaluation*cluster.cpuspernode+1;
+		end
+		%}}}
+function BuildQueueScript(cluster,dirname,modelname,solution,io_gather,isvalgrind,isgprof,isdakota,isoceancoupling) % {{{
+
+			executable='issm.exe';
+			if isdakota,
+				version=IssmConfig('_DAKOTA_VERSION_'); version=str2num(version(1:3));
+				if (version>=6),
+					executable='issm_dakota.exe';
+				end
+			end
+
+			%write queuing script 
+			fid=fopen([modelname '.queue'],'w');
+			fprintf(fid,'#PBS -S /bin/bash\n');
+			fprintf(fid,'#PBS -l select=1:ncpus=%i:model=%s+%i:ncpus=%i:model=%s\n',cluster.cpuspernode+1,cluster.processor,cluster.concurrent_evaluations*cluster.numnodes_per_evaluation-1,...
+																					cluster.cpuspernode,cluster.processor);
+			fprintf(fid,'#PBS -l walltime=%i\n',cluster.time*60); %walltime is in seconds.
+			fprintf(fid,'#PBS -q %s\n',cluster.queue);
+			fprintf(fid,'#PBS -W group_list=%s\n',cluster.grouplist);
+			fprintf(fid,'#PBS -m e\n');
+			fprintf(fid,'#PBS -o %s/%s/%s.outlog \n',cluster.executionpath,dirname,modelname);
+			fprintf(fid,'#PBS -e %s/%s/%s.errlog \n\n',cluster.executionpath,dirname,modelname);
+			fprintf(fid,'. /usr/share/modules/init/bash\n\n');
+			for i=1:numel(cluster.modules), fprintf(fid,['module load ' cluster.modules{i} '\n']); end
+			fprintf(fid,'export PATH="$PATH:."\n\n');
+			fprintf(fid,'export MPI_LAUNCH_TIMEOUT=520\n');
+			fprintf(fid,'export MPI_GROUP_MAX=64\n\n');
+			fprintf(fid,'export ISSM_DIR="%s"\n',cluster.srcpath); %FIXME
+			fprintf(fid,'source $ISSM_DIR/etc/environment.sh\n');       %FIXME
+			fprintf(fid,'cd %s/%s/\n\n',cluster.executionpath,dirname);
+			fprintf(fid,'/u/scicon/tools/bin/several_tries mpiexec -np %i /u/scicon/tools/bin/mbind.x -v  %s/%s %s %s/%s %s\n',cluster.concurrent_evaluations*cluster.numnodes_per_evaluation*cluster.cpuspernode+1,...
+																																	 cluster.codepath,executable,solution,cluster.executionpath,dirname,modelname);
+			if ~io_gather, %concatenate the output files:
+				fprintf(fid,'cat %s.outbin.* > %s.outbin',modelname,modelname);
+			end
+			fclose(fid);
+
+			%in interactive mode, create a run file, and errlog and outlog file
+			if cluster.interactive,
+				fid=fopen([modelname '.run'],'w');
+				if cluster.interactive==10,
+						fprintf(fid,'module unload mpi-mvapich2/1.4.1/gcc\n');
+						fprintf(fid,'/u/scicon/tools/bin/several_tries mpiexec -np %i /u/scicon/tools/bin/mbind.x -v  %s/%s %s %s/Interactive%i %s\n',cluster.concurrent_evaluations*cluster.numnodes_per_evaluation*cluster.cpuspernode+1,...
+																																	 cluster.codepath,executable,solution,cluster.executionpath,cluster.interactive,modelname);
+				else
+						fprintf(fid,'mpiexec -np %i %s/%s %s %s %s\n',cluster.nprocs(),cluster.codepath,executable,solution,[cluster.executionpath '/Interactive' num2str(cluster.interactive)],modelname);
+				end
+				if ~io_gather, %concatenate the output files:
+					fprintf(fid,'cat %s.outbin.* > %s.outbin',modelname,modelname);
+				end
+				fclose(fid);
+				fid=fopen([modelname '.errlog'],'w');
+				fclose(fid);
+				fid=fopen([modelname '.outlog'],'w');
+				fclose(fid);
+			end
+		end %}}}
+		function UploadQueueJob(cluster,modelname,dirname,filelist) % {{{
+
+			%compress the files into one zip.
+			compressstring=['tar -zcf ' dirname '.tar.gz'];
+			for i=1:numel(filelist),
+				compressstring = [compressstring ' ' filelist{i}];
+			end
+			if cluster.interactive,
+				compressstring = [compressstring ' ' modelname '.run '  modelname '.errlog ' modelname '.outlog '];
+			end
+			system(compressstring);
+
+			disp('uploading input file and queueing script');
+			if cluster.interactive==10,
+				directory=[pwd() '/run/'];
+			elseif cluster.interactive,
+				directory=[cluster.executionpath '/Interactive' num2str(cluster.interactive)];
+			else 
+				directory=cluster.executionpath;
+			end
+
+			if cluster.bbftp,
+				issmbbftpout(cluster.name,directory,cluster.login,cluster.port,cluster.numstreams,{[dirname '.tar.gz']});
+			else
+				issmscpout(cluster.name,directory,cluster.login,cluster.port,{[dirname '.tar.gz']});
+			end
+
+		end
+		%}}}
+		function LaunchQueueJob(cluster,modelname,dirname,filelist,restart,batch) % {{{
+
+			%launch command, to be executed via ssh
+			if cluster.interactive,
+				if ~isempty(restart)
+					launchcommand=['cd ' cluster.executionpath '/Interactive' num2str(cluster.interactive)];
+				else
+					if cluster.interactive==10,
+						launchcommand=['cd ' pwd() '/run && tar -zxf ' dirname '.tar.gz'];
+					else
+						launchcommand=['cd ' cluster.executionpath '/Interactive' num2str(cluster.interactive) ' && tar -zxf ' dirname '.tar.gz'];
+					end
+				end
+			else
+				if ~isempty(restart)
+					launchcommand=['cd ' cluster.executionpath ' && cd ' dirname ' && qsub ' modelname '.queue '];
+				else
+					launchcommand=['cd ' cluster.executionpath ' && rm -rf ./' dirname ' && mkdir ' dirname ...
+						' && cd ' dirname ' && mv ../' dirname '.tar.gz ./ && tar -zxf ' dirname '.tar.gz && qsub ' modelname '.queue '];
+				end
+			end
+
+			disp('launching solution sequence on remote cluster');
+			issmssh(cluster.name,cluster.login,cluster.port,launchcommand);
+		end
+		%}}}
+		function Download(cluster,dirname,filelist) % {{{
+
+			%copy files from cluster to current directory
+			if cluster.interactive==10,
+				directory=[pwd() '/run/'];
+			elseif ~cluster.interactive,
+				directory=[cluster.executionpath '/' dirname '/'];
+			else
+				directory=[cluster.executionpath '/Interactive' num2str(cluster.interactive) '/'];
+			end
+
+			if cluster.bbftp,
+				issmbbftpin(cluster.name, cluster.login, cluster.port, cluster.numstreams, directory, filelist);
+			else
+				issmscpin(cluster.name,cluster.login,cluster.port,directory,filelist);
+			end
+
+		end %}}}
+	end
+end