Index: /issm/trunk-jpl/test/MITgcm/code/CPP_EEOPTIONS.h =================================================================== --- /issm/trunk-jpl/test/MITgcm/code/CPP_EEOPTIONS.h (revision 22681) +++ /issm/trunk-jpl/test/MITgcm/code/CPP_EEOPTIONS.h (revision 22681) @@ -0,0 +1,167 @@ +C $Header: /u/gcmpack/MITgcm/eesupp/inc/CPP_EEOPTIONS.h,v 1.43 2017/09/14 19:47:35 jmc Exp $ +C $Name: $ + +CBOP +C !ROUTINE: CPP_EEOPTIONS.h +C !INTERFACE: +C include "CPP_EEOPTIONS.h" +C +C !DESCRIPTION: +C *==========================================================* +C | CPP\_EEOPTIONS.h | +C *==========================================================* +C | C preprocessor "execution environment" supporting | +C | flags. Use this file to set flags controlling the | +C | execution environment in which a model runs - as opposed | +C | to the dynamical problem the model solves. | +C | Note: Many options are implemented with both compile time| +C | and run-time switches. This allows options to be | +C | removed altogether, made optional at run-time or | +C | to be permanently enabled. This convention helps | +C | with the data-dependence analysis performed by the | +C | adjoint model compiler. This data dependency | +C | analysis can be upset by runtime switches that it | +C | is unable to recoginise as being fixed for the | +C | duration of an integration. | +C | A reasonable way to use these flags is to | +C | set all options as selectable at runtime but then | +C | once an experimental configuration has been | +C | identified, rebuild the code with the appropriate | +C | options set at compile time. | +C *==========================================================* +CEOP + +#ifndef _CPP_EEOPTIONS_H_ +#define _CPP_EEOPTIONS_H_ + +C In general the following convention applies: +C ALLOW - indicates an feature will be included but it may +C CAN have a run-time flag to allow it to be switched +C on and off. +C If ALLOW or CAN directives are "undef'd" this generally +C means that the feature will not be available i.e. it +C will not be included in the compiled code and so no +C run-time option to use the feature will be available. +C +C ALWAYS - indicates the choice will be fixed at compile time +C so no run-time option will be present + +C=== Macro related options === +C-- Control storage of floating point operands +C On many systems it improves performance only to use +C 8-byte precision for time stepped variables. +C Constant in time terms ( geometric factors etc.. ) +C can use 4-byte precision, reducing memory utilisation and +C boosting performance because of a smaller working set size. +C However, on vector CRAY systems this degrades performance. +C Enable to switch REAL4_IS_SLOW from genmake2 (with LET_RS_BE_REAL4): +#ifdef LET_RS_BE_REAL4 +#undef REAL4_IS_SLOW +#else /* LET_RS_BE_REAL4 */ +#define REAL4_IS_SLOW +#endif /* LET_RS_BE_REAL4 */ + +C-- Control use of "double" precision constants. +C Use D0 where it means REAL*8 but not where it means REAL*16 +#define D0 d0 + +C-- Enable some old macro conventions for backward compatibility +#undef USE_OLD_MACROS_R4R8toRSRL + +C=== IO related options === +C-- Flag used to indicate whether Fortran formatted write +C and read are threadsafe. On SGI the routines can be thread +C safe, on Sun it is not possible - if you are unsure then +C undef this option. +#undef FMTFTN_IO_THREAD_SAFE + +C-- Flag used to indicate whether Binary write to Local file (i.e., +C a different file for each tile) and read are thread-safe. +#undef LOCBIN_IO_THREAD_SAFE + +C-- Flag to turn off the writing of error message to ioUnit zero +#undef DISABLE_WRITE_TO_UNIT_ZERO + +C-- Alternative formulation of BYTESWAP, faster than +C compiler flag -byteswapio on the Altix. +#undef FAST_BYTESWAP + +C-- Flag to turn on old default of opening scratch files with the +C STATUS='SCRATCH' option. This method, while perfectly FORTRAN-standard, +C caused filename conflicts on some multi-node/multi-processor platforms +C in the past and has been replace by something (hopefully) more robust. +#undef USE_FORTRAN_SCRATCH_FILES + +C-- Flag defined for eeboot_minimal.F, eeset_parms.F and open_copy_data_file.F +C to write STDOUT, STDERR and scratch files from process 0 only. +C WARNING: to use only when absolutely confident that the setup is working +C since any message (error/warning/print) from any proc <> 0 will be lost. +#undef SINGLE_DISK_IO + +C=== MPI, EXCH and GLOBAL_SUM related options === +C-- Flag turns off MPI_SEND ready_to_receive polling in the +C gather_* subroutines to speed up integrations. +#undef DISABLE_MPI_READY_TO_RECEIVE + +C-- Control MPI based parallel processing +CXXX We no longer select the use of MPI via this file (CPP_EEOPTIONS.h) +CXXX To use MPI, use an appropriate genmake2 options file or use +CXXX genmake2 -mpi . +CXXX #undef ALLOW_USE_MPI + +C-- Control use of communication that might overlap computation. +C Under MPI selects/deselects "non-blocking" sends and receives. +#define ALLOW_ASYNC_COMMUNICATION +#undef ALLOW_ASYNC_COMMUNICATION +#undef ALWAYS_USE_ASYNC_COMMUNICATION +C-- Control use of communication that is atomic to computation. +C Under MPI selects/deselects "blocking" sends and receives. +#define ALLOW_SYNC_COMMUNICATION +#undef ALWAYS_USE_SYNC_COMMUNICATION + +C-- Control XY periodicity in processor to grid mappings +C Note: Model code does not need to know whether a domain is +C periodic because it has overlap regions for every box. +C Model assume that these values have been +C filled in some way. +#undef ALWAYS_PREVENT_X_PERIODICITY +#undef ALWAYS_PREVENT_Y_PERIODICITY +#define CAN_PREVENT_X_PERIODICITY +#define CAN_PREVENT_Y_PERIODICITY + +C-- disconnect tiles (no exchange between tiles, just fill-in edges +C assuming locally periodic subdomain) +#undef DISCONNECTED_TILES + +C-- Always cumulate tile local-sum in the same order by applying MPI allreduce +C to array of tiles ; can get slower with large number of tiles (big set-up) +#define GLOBAL_SUM_ORDER_TILES + +C-- Alternative way of doing global sum without MPI allreduce call +C but instead, explicit MPI send & recv calls. Expected to be slower. +#undef GLOBAL_SUM_SEND_RECV + +C-- Alternative way of doing global sum on a single CPU +C to eliminate tiling-dependent roundoff errors. Note: This is slow. +#undef CG2D_SINGLECPU_SUM + +C=== Other options (to add/remove pieces of code) === +C-- Flag to turn on checking for errors from all threads and procs +C (calling S/R STOP_IF_ERROR) before stopping. +#define USE_ERROR_STOP + +C-- Control use of communication with other component: +C allow to import and export from/to Coupler interface. +#undef COMPONENT_MODULE + +C-- Options used to couple MITgcm and ISSM +C Eventually this option can probably be merged with COMPONENT_MODULE +#define ALLOW_CPL_ISSM + +C-- Activate some pieces of code for coupling to GEOS AGCM +#undef HACK_FOR_GMAO_CPL + +#endif /* _CPP_EEOPTIONS_H_ */ + +#include "CPP_EEMACROS.h" + Index: /issm/trunk-jpl/test/MITgcm/code/cpl_issm.F =================================================================== --- /issm/trunk-jpl/test/MITgcm/code/cpl_issm.F (revision 22680) +++ /issm/trunk-jpl/test/MITgcm/code/cpl_issm.F (revision 22681) @@ -1,6 +1,4 @@ #include "PACKAGES_CONFIG.h" #include "CPP_OPTIONS.h" - -#define ALLOW_CPL_ISSM CBOP Index: /issm/trunk-jpl/test/MITgcm/code/do_oceanic_phys.F =================================================================== --- /issm/trunk-jpl/test/MITgcm/code/do_oceanic_phys.F (revision 22680) +++ /issm/trunk-jpl/test/MITgcm/code/do_oceanic_phys.F (revision 22681) @@ -534,7 +534,6 @@ #endif /* OLD_THSICE_CALL_SEQUENCE */ -#define ALLOW_CPL_ISSM #ifdef ALLOW_CPL_ISSM - CALL CPL_ISSM( myTime, myIter, myThid ) + IF ( useCoupler) CALL CPL_ISSM( myTime, myIter, myThid ) #endif Index: /issm/trunk-jpl/test/MITgcm/code/eeboot_minimal.F =================================================================== --- /issm/trunk-jpl/test/MITgcm/code/eeboot_minimal.F (revision 22680) +++ /issm/trunk-jpl/test/MITgcm/code/eeboot_minimal.F (revision 22681) @@ -4,6 +4,4 @@ #include "PACKAGES_CONFIG.h" #include "CPP_EEOPTIONS.h" - -#define ALLOW_CPL_ISSM CBOP @@ -69,5 +67,4 @@ INTEGER mpiMyWid, toissmcomm INTEGER my_local_rank,my_local_size, numprocsworld - INTEGER dummy1(1),dummy2(1) INTEGER status(MPI_STATUS_SIZE) #endif /* ALLOW_CPL_ISSM */ @@ -183,36 +180,29 @@ #ifdef ALLOW_CPL_ISSM - call MPI_COMM_RANK(MPI_COMM_WORLD, mpiMyWid, mpiRC) - call MPI_COMM_SIZE(MPI_COMM_WORLD, numprocsworld, mpiRC) +C add a 1rst preliminary call EESET_PARAMS to set useCoupler + CALL MPI_COMM_RANK(MPI_COMM_WORLD, mpiMyWid, mpiRC) + CALL EESET_PARMS ( mpiMyWId, doReport ) + + IF ( useCoupler ) THEN + CALL MPI_COMM_SIZE(MPI_COMM_WORLD, numprocsworld, mpiRC) c Split world into sub-communicators for each and every model:*/ - call MPI_COMM_SPLIT(MPI_COMM_WORLD,1,MPIMYWID, - & MPI_COMM_MODEL,mpiRC) - - print*,'Oc My global rank',mpiMyWid - print*,'Oc My world size:',numprocsworld - - call MPI_INTERCOMM_CREATE(MPI_COMM_MODEL,0,MPI_COMM_WORLD, - & 0,0,toissmcomm,mpiRC) - - call MPI_COMM_RANK(MPI_COMM_MODEL, my_local_rank, mpiRC) - call MPI_COMM_SIZE(MPI_COMM_MODEL, my_local_size, mpiRC) - - print*,'Oc My global rank',mpiMyWid,'MyLocal rank: ',my_local_rank - print*,'Oc My world size:',numprocsworld,'My local size: ', - & my_local_size - - -c receive and send an integer: - dummy1(1)=4; dummy2(1)=2; - - if (my_local_rank .eq. 0) then -cdb call MPI_Send(dummy2,1,MPI_INT,0,1,toissmcomm,mpiRC) -cdb call MPI_Recv(dummy1,1,MPI_INT,0,1,toissmcomm,status,mpiRC) - end if - -cdb call MPI_Bcast(dummy1,1,MPI_INT,0,MPI_COMM_MODEL,mpiRC) -cdb print*, 'Ocean : dummy received:',dummy1 - + CALL MPI_COMM_SPLIT(MPI_COMM_WORLD,1,MPIMYWID, + & MPI_COMM_MODEL,mpiRC) + + print*,'Oc My global rank',mpiMyWid + print*,'Oc My world size:',numprocsworld + + CALL MPI_INTERCOMM_CREATE(MPI_COMM_MODEL,0,MPI_COMM_WORLD, + & 0,0,toissmcomm,mpiRC) + + CALL MPI_COMM_RANK(MPI_COMM_MODEL, my_local_rank, mpiRC) + CALL MPI_COMM_SIZE(MPI_COMM_MODEL, my_local_size, mpiRC) + + print*,'Oc My global rank',mpiMyWid,'MyLocal rank: ', + & my_local_rank + print*,'Oc My world size:',numprocsworld,'My local size: ', + & my_local_size + ENDIF #endif /* ALLOW_CPL_ISSM */ Index: /issm/trunk-jpl/test/MITgcm/code/eedie.F =================================================================== --- /issm/trunk-jpl/test/MITgcm/code/eedie.F (revision 22680) +++ /issm/trunk-jpl/test/MITgcm/code/eedie.F (revision 22681) @@ -88,5 +88,5 @@ C-- MPI style multiple-process termination C-- ====================================== -#ifdef COMPONENT_MODULE +#if (defined COMPONENT_MODULE) || (defined ALLOW_CPL_ISSM) IF ( useCoupler) CALL MPI_BARRIER( MPI_COMM_WORLD, mpiRC ) #endif @@ -94,5 +94,4 @@ IF ( useOASIS ) CALL OASIS_FINALIZE #endif - CALL MPI_BARRIER( MPI_COMM_WORLD, mpiRC ) CALL MPI_FINALIZE ( mpiRC ) IF ( mpiRC .NE. MPI_SUCCESS ) THEN