Add pio_finalize calls for CICE history and restart files (#995)

* Add pio_finalize calls for CICE history and restart files when running with PIO.  This
cleans up some PIO internal data.  The code was aborting when trying to write
lots of files (>600) because we reinitialize PIO for each file.  This is a short term
fix.  A longer term fix is to initialize PIO once and reuse as much as possible.

Add a new test that runs CICE for a year writing hourly files (>8000 files).  Test with
pio2 and netcdf.

* Add error checks and CESMCOUPLED cpp to new ice_pio_finalize method.

Author: apcraig

cicecore/cicedyn/infrastructure/io/io_pio2/ice_history_write.F90

@@ -1255,6 +1255,12 @@ subroutine ice_write_hist (ns)
          write(nu_diag,*) 'Finished writing ',trim(ncfile)
       endif
 
+      !-----------------------------------------------------------------
+      ! clean up PIO
+      !-----------------------------------------------------------------
+
+      call ice_pio_finalize()
+
       first_call = .false.
 
       end subroutine ice_write_hist

cicecore/cicedyn/infrastructure/io/io_pio2/ice_pio.F90

@@ -27,6 +27,7 @@ module ice_pio
 
   public ice_pio_init
   public ice_pio_initdecomp
+  public ice_pio_finalize
   public ice_pio_check
 
 #ifdef CESMCOUPLED
@@ -529,6 +530,27 @@ subroutine ice_pio_initdecomp_4d (ndim3, ndim4, iodesc, precision)
    end subroutine ice_pio_initdecomp_4d
 
 
+!================================================================================
+
+   ! PIO Finalize
+
+   subroutine ice_pio_finalize()
+
+      integer(kind=int_kind)      :: status
+      character(len=*), parameter :: subname = '(ice_pio_finalize)'
+
+      status = PIO_NOERR
+#ifndef CESMCOUPLED
+      call pio_seterrorhandling(ice_pio_subsystem, PIO_RETURN_ERROR)
+      call pio_finalize(ice_pio_subsystem,status)
+      call ice_pio_check( status, subname//' ERROR: Failed to finalize ', &
+         file=__FILE__,line=__LINE__)
+! do not call this, ice_pio_subsystem does not exist anymore
+!      call pio_seterrorhandling(ice_pio_subsystem, PIO_INTERNAL_ERROR)
+#endif
+
+   end subroutine ice_pio_finalize
+
 !================================================================================
 
    ! PIO Error handling

cicecore/cicedyn/infrastructure/io/io_pio2/ice_restart.F90

@@ -905,6 +905,7 @@ subroutine final_restart()
       call PIO_freeDecomp(File,iodesc2d)
       call PIO_freeDecomp(File,iodesc3d_ncat)
       call pio_closefile(File)
+      call ice_pio_finalize()
 
       if (my_task == master_task) then
          write(nu_diag,'(a,i8,4x,i4.4,a,i2.2,a,i2.2,a,i5.5)') &

configuration/scripts/options/set_nml.histhrly

@@ -0,0 +1,8 @@
+    histfreq       = 'm','h','x','x','x'
+    histfreq_n     =  1,1,1,1,1
+    histfreq_base  =  'zero','zero','zero','zero','zero'
+    write_ic       = .true.
+    f_aice         = 'mh' 
+    f_hi           = 'hm'
+    f_hs           = 'mh' 
+    f_Tsfc         = 'mh' 

configuration/scripts/tests/io_suite.ts

@@ -1,4 +1,7 @@
 # Test         Grid    PEs        Sets    BFB-compare
+smoke          gx3    32x1        run1year,histhrly,ionetcdf,iocdf2,short
+smoke          gx3    32x1        run1year,histhrly,iopio2,iocdf5,short
+
 # some iobinary configurations fail due to bathymetry netcdf file requirement, remove them
 # iobinary cannot work with JRA55 because netcdf is turned off
 restart        gx3     8x4        gx3ncarbulk,debug,histall,iobinary,precision8