MOM_write_cputime.F90

! This file is part of MOM6, the Modular Ocean Model version 6.

! See the LICENSE file for licensing information.

! SPDX-License-Identifier: Apache-2.0

!> A module to monitor the overall CPU time used by MOM6 and project when to stop the model

module mom_write_cputime

use mom_coms,          only : sum_across_pes, num_pes

use mom_error_handler, only : mom_error, mom_mesg, fatal, is_root_pe

use mom_io,            only : open_ascii_file, close_file, append_file, writeonly_file

use mom_file_parser,   only : get_param, log_param, log_version, param_file_type

use mom_time_manager,  only : time_type, get_time, operator(>)

implicit none ; private

public write_cputime, mom_write_cputime_init, mom_write_cputime_end, write_cputime_start_clock

!-----------------------------------------------------------------------

integer :: clocks_per_sec = 1000 !< The number of clock cycles per second, used by the system clock

integer :: max_ticks      = 1000 !< The number of ticks per second, used by the system clock

!> A control structure that regulates the writing of CPU time

type, public :: write_cputime_cs ; private

  logical :: initialized = .false. !< True if this control structure has been initialized.

  real :: maxcpu                !<   The maximum amount of CPU time per processor

                                !! for which MOM should run before saving a restart

                                !! file and quitting with a return value that

                                !! indicates that further execution is required to

                                !! complete the simulation [wall-clock seconds].

  type(time_type) :: start_time !< The start time of the simulation.

                                !! Start_time is set in MOM_initialization.F90

  real :: startup_cputime       !< The CPU time used in the startup phase of the model [clock_cycles].

  real :: prev_cputime = 0.0    !< The last measured CPU time [clock_cycles].

  real :: dn_dcpu_min = -1.0    !< The minimum derivative of timestep with CPU time [steps clock_cycles-1].

  real :: cputime2 = 0.0        !< The accumulated CPU time [clock_cycles].

  integer :: previous_calls = 0 !< The number of times write_CPUtime has been called.

  integer :: prev_n = 0         !< The value of n from the last call.

  integer :: filecpu_ascii= -1  !< The unit number of the CPU time file.

  character(len=200) :: cpufile !< The name of the CPU time file.

end type write_cputime_cs

contains

!> Evaluate the CPU time returned by SYSTEM_CLOCK at the start of a run

subroutine write_cputime_start_clock(CS)

  type(write_cputime_cs), pointer :: cs !< The control structure set up by a previous

                                        !! call to MOM_write_cputime_init.

  integer :: new_cputime   ! The CPU time returned by SYSTEM_CLOCK

  if (.not.associated(cs)) allocate(cs)

  call system_clock(new_cputime, clocks_per_sec, max_ticks)

  cs%prev_cputime = new_cputime

end subroutine write_cputime_start_clock

!> Initialize the MOM_write_cputime module.

subroutine mom_write_cputime_init(param_file, directory, Input_start_time, CS)

  type(param_file_type),  intent(in) :: param_file !< A structure to parse for run-time parameters

  character(len=*),       intent(in) :: directory  !< The directory where the CPU time file goes.

  type(time_type),        intent(in) :: input_start_time !< The start model time of the simulation.

  type(write_cputime_cs), pointer    :: cs         !< A pointer that may be set to point to the

                                                   !! control structure for this module.

  ! Local variables

  integer :: new_cputime   ! The CPU time returned by SYSTEM_CLOCK

  ! This include declares and sets the variable "version".

# include "version_variable.h"

  character(len=40)  :: mdl = 'MOM_write_cputime'  ! This module's name.

  logical :: all_default   ! If true, all parameters are using their default values.

  if (.not.associated(cs)) then

    allocate(cs)

    call system_clock(new_cputime, clocks_per_sec, max_ticks)

    cs%prev_cputime = new_cputime

  endif

  cs%initialized = .true.

  ! Read all relevant parameters and write them to the model log.

  ! Determine whether all parameters are set to their default values.

  call get_param(param_file, mdl, "MAXCPU", cs%maxcpu, units="wall-clock seconds", default=-1.0, do_not_log=.true.)

  call get_param(param_file, mdl, "CPU_TIME_FILE", cs%CPUfile, default="CPU_stats", do_not_log=.true.)

  all_default = (cs%maxcpu == -1.0) .and. (trim(cs%CPUfile) == trim("CPU_stats"))

  call log_version(param_file, mdl, version, "", all_default=all_default)

  call get_param(param_file, mdl, "MAXCPU", cs%maxcpu, &

                 "The maximum amount of cpu time per processor for which "//&

                 "MOM should run before saving a restart file and "//&

                 "quitting with a return value that indicates that a "//&

                 "further run is required to complete the simulation. "//&

                 "If automatic restarts are not desired, use a negative "//&

                 "value for MAXCPU.  MAXCPU has units of wall-clock "//&

                 "seconds, so the actual CPU time used is larger by a "//&

                 "factor of the number of processors used.", &

                 units="wall-clock seconds", default=-1.0)

  call get_param(param_file, mdl, "CPU_TIME_FILE", cs%CPUfile, &

                 "The file into which CPU time is written.",default="CPU_stats")

  cs%CPUfile = trim(directory)//trim(cs%CPUfile)

  call log_param(param_file, mdl, "directory/CPU_TIME_FILE", cs%CPUfile)

#ifdef STATSLABEL

  cs%CPUfile = trim(cs%CPUfile)//"."//trim(adjustl(statslabel))

#endif

  cs%Start_time = input_start_time

end subroutine mom_write_cputime_init

!> Close the MOM_write_cputime module.

subroutine mom_write_cputime_end(CS)

  type(write_cputime_cs), pointer    :: cs    !< The control structure set up by a previous

                                              !! call to MOM_write_cputime_init.

  if (.not.associated(cs)) return

  ! Flush and close the output files.

  if (is_root_pe() .and. cs%fileCPU_ascii > 0) then

    flush(cs%fileCPU_ascii)

    call close_file(cs%fileCPU_ascii)

  endif

  deallocate(cs)

end subroutine mom_write_cputime_end

!> This subroutine assesses how much CPU time the model has taken and determines how long the model

!! should be run before it saves a restart file and stops itself.  Optionally this may also be used

!! to trigger this module's end routine.

subroutine write_cputime(day, n, CS, nmax, call_end)

  type(time_type),        intent(inout) :: day  !< The current model time.

  integer,                intent(in)    :: n    !< The time step number of the current execution.

  type(write_cputime_cs), pointer       :: cs   !< The control structure set up by a previous

                                                !! call to MOM_write_cputime_init.

  integer,      optional, intent(inout) :: nmax !< The number of iterations after which to stop so

                                                !! that the simulation will not run out of CPU time.

  logical,      optional, intent(in)    :: call_end !< If true, also call MOM_write_cputime_end.

  ! Local variables

  real    :: d_cputime     ! The change in CPU time since the last call

                           ! this subroutine [clock_cycles]

  integer :: new_cputime   ! The CPU time returned by SYSTEM_CLOCK [clock_cycles]

  real    :: reday         ! The time in days, including fractional days [days]

  integer :: start_of_day  ! The number of seconds since the start of the day

  integer :: num_days      ! The number of days in the time

  if (.not.associated(cs)) call mom_error(fatal, &

         "write_energy: Module must be initialized before it is used.")

  if (.not.cs%initialized) call mom_error(fatal, &

         "write_cputime: Module must be initialized before it is used.")

  call system_clock(new_cputime, clocks_per_sec, max_ticks)

!   The following lines extract useful information even if the clock has rolled

! over, assuming a 32-bit SYSTEM_CLOCK.  With more bits, rollover is essentially

! impossible. Negative fluctuations of less than 10 seconds are not interpreted

! as the clock rolling over.  This should be unnecessary but is sometimes needed

! on the GFDL SGI/O3k.

  if (new_cputime < cs%prev_cputime-(10.0*clocks_per_sec)) then

    d_cputime = new_cputime - cs%prev_cputime + max_ticks

  else

    d_cputime = new_cputime - cs%prev_cputime

  endif

  call sum_across_pes(d_cputime)

  if (cs%previous_calls == 0) cs%startup_cputime = d_cputime

  cs%cputime2 = cs%cputime2 + d_cputime

  if ((cs%previous_calls >= 1) .and. (cs%maxcpu > 0.0)) then

    ! Determine the slowest rate at which time steps are executed.

    if ((n > cs%prev_n) .and. (d_cputime > 0.0) .and. &

        ((cs%dn_dcpu_min*d_cputime < (n - cs%prev_n)) .or. &

         (cs%dn_dcpu_min < 0.0))) &

      cs%dn_dcpu_min = (n - cs%prev_n) / d_cputime

    if (present(nmax) .and. (cs%dn_dcpu_min >= 0.0)) then

      ! Have the model stop itself after 95% of the CPU time has been used.

      nmax = n + int( cs%dn_dcpu_min * &

          (0.95*cs%maxcpu * real(num_pes())*clocks_per_sec - &

           (cs%startup_cputime + cs%cputime2)) )

!     write(mesg,*) "Resetting nmax to ",nmax," at day",reday

!     call MOM_mesg(mesg)

    endif

  endif

  cs%prev_cputime = new_cputime ; cs%prev_n = n

  call get_time(day, start_of_day, num_days)

  reday = real(num_days)+ (real(start_of_day)/86400.0)

  !  Reopen or create a text output file.

  if ((cs%previous_calls == 0) .and. (is_root_pe())) then

    if (day > cs%Start_time) then

      call open_ascii_file(cs%fileCPU_ascii, trim(cs%CPUfile), action=append_file)

    else

      call open_ascii_file(cs%fileCPU_ascii, trim(cs%CPUfile), action=writeonly_file)

    endif

  endif

  if (is_root_pe()) then

    if (cs%previous_calls == 0) then

      write(cs%fileCPU_ascii, &

        '("Startup CPU time: ", F12.3, " sec summed across", I5, " PEs.")') &

                            (cs%startup_cputime / clocks_per_sec), num_pes()

      write(cs%fileCPU_ascii,*)"        Day, Step number,     CPU time, CPU time change"

    endif

    write(cs%fileCPU_ascii,'(F12.3,", ",I11,", ",F12.3,", ",F12.3)') &

           reday, n, (cs%cputime2 / real(clocks_per_sec)), &

           d_cputime / real(clocks_per_sec)

    flush(cs%fileCPU_ascii)

  endif

  cs%previous_calls = cs%previous_calls + 1

  if (present(call_end)) then

    if (call_end) call mom_write_cputime_end(cs)

  endif

end subroutine write_cputime

!> \namespace mom_write_cputime

!!

!!  By Robert Hallberg, May 2006.

!!

!!    This file contains the subroutine (write_cputime) that writes

!!  the summed CPU time across all processors to an output file. In

!!  addition, write_cputime estimates how many more time steps can be

!!  taken before 95% of the available CPU time is used, so that the

!!  model can be checkpointed at that time.

end module mom_write_cputime