From 4bf2611e788342a40d7a759878b26da6706851c8 Mon Sep 17 00:00:00 2001 From: apcraig Date: Mon, 14 Mar 2022 11:51:17 -0600 Subject: [PATCH 1/2] - Work around a couple C/CD issues, https://github.com/CICE-Consortium/CICE/issues/700 - modify gridcell + land block elimination for C/CD, using a larger stencil, needs to be investigate further - turn off maskhalo_dyn for C/CD, needs to be investigated further - Update omp_suite and gridsys_suite to extend testing - Add new histdbg option to turn on some output at each timestep to help debugging --- cicecore/cicedynB/dynamics/ice_dyn_evp.F90 | 4 +- .../cicedynB/infrastructure/ice_domain.F90 | 35 ++- cicecore/cicedynB/infrastructure/ice_grid.F90 | 2 +- configuration/scripts/options/set_nml.histdbg | 225 ++++++++++++++++++ configuration/scripts/tests/gridsys_suite.ts | 24 ++ configuration/scripts/tests/omp_suite.ts | 9 + 6 files changed, 288 insertions(+), 11 deletions(-) create mode 100644 configuration/scripts/options/set_nml.histdbg diff --git a/cicecore/cicedynB/dynamics/ice_dyn_evp.F90 b/cicecore/cicedynB/dynamics/ice_dyn_evp.F90 index 360781a79..42fe1b53d 100644 --- a/cicecore/cicedynB/dynamics/ice_dyn_evp.F90 +++ b/cicecore/cicedynB/dynamics/ice_dyn_evp.F90 @@ -971,7 +971,9 @@ subroutine evp (dt) call ice_timer_start(timer_bound) call stack_velocity_field(uvel, vvel, fld2) - if (maskhalo_dyn) then + ! maskhalo_dyn causes non bit-for-bit results on different decomps + ! with C/CD in some cases + if (grid_ice == 'B' .and. maskhalo_dyn) then call ice_HaloUpdate (fld2, halo_info_mask, & field_loc_NEcorner, field_type_vector) else diff --git a/cicecore/cicedynB/infrastructure/ice_domain.F90 b/cicecore/cicedynB/infrastructure/ice_domain.F90 index 6f8fee49a..0d0502e85 100644 --- a/cicecore/cicedynB/infrastructure/ice_domain.F90 +++ b/cicecore/cicedynB/infrastructure/ice_domain.F90 @@ -287,7 +287,7 @@ end subroutine init_domain_blocks !*********************************************************************** - subroutine init_domain_distribution(KMTG,ULATG) + subroutine init_domain_distribution(KMTG,ULATG,grid_ice) ! This routine calls appropriate setup routines to distribute blocks ! across processors and defines arrays with block ids for any local @@ -302,6 +302,9 @@ subroutine init_domain_distribution(KMTG,ULATG) KMTG ,&! global topography ULATG ! global latitude field (radians) + character(len=*), intent(in) :: & + grid_ice ! grid_ice, B, C, CD, etc + !---------------------------------------------------------------------- ! ! local variables @@ -319,6 +322,7 @@ subroutine init_domain_distribution(KMTG,ULATG) integer (int_kind) :: & i,j,n ,&! dummy loop indices ig,jg ,&! global indices + igm1,igp1,jgm1,jgp1,&! global indices ninfo ,&! ice_distributionGet check work_unit ,&! size of quantized work unit #ifdef USE_NETCDF @@ -509,10 +513,25 @@ subroutine init_domain_distribution(KMTG,ULATG) if (this_block%i_glob(i) > 0) then ig = this_block%i_glob(i) jg = this_block%j_glob(j) - if (KMTG(ig,jg) > puny .and. & - (ULATG(ig,jg) < shlat/rad_to_deg .or. & - ULATG(ig,jg) > nhlat/rad_to_deg) ) & - nocn(n) = nocn(n) + flat(ig,jg) + if (grid_ice == 'C' .or. grid_ice == 'CD') then + ! Have to be careful about block elimination with C/CD + ! Use a bigger stencil + igm1 = mod(ig-2+nx_global,nx_global)+1 + igp1 = mod(ig,nx_global)+1 + jgm1 = max(jg-1,1) + jgp1 = min(jg+1,ny_global) + if ((KMTG(ig ,jg ) > puny .or. & + KMTG(igm1,jg ) > puny .or. KMTG(igp1,jg ) > puny .or. & + KMTG(ig ,jgp1) > puny .or. KMTG(ig ,jgm1) > puny) .and. & + (ULATG(ig,jg) < shlat/rad_to_deg .or. & + ULATG(ig,jg) > nhlat/rad_to_deg) ) & + nocn(n) = nocn(n) + flat(ig,jg) + else + if (KMTG(ig,jg) > puny .and. & + (ULATG(ig,jg) < shlat/rad_to_deg .or. & + ULATG(ig,jg) > nhlat/rad_to_deg) ) & + nocn(n) = nocn(n) + flat(ig,jg) + endif endif end do endif @@ -529,10 +548,8 @@ subroutine init_domain_distribution(KMTG,ULATG) ! Keep all blocks even the ones only containing land points if (distribution_wght == 'block') nocn(n) = nx_block*ny_block #else - if (distribution_wght == 'block' .and. & ! POP style - nocn(n) > 0) nocn(n) = nx_block*ny_block - if (distribution_wght == 'blockall') & - nocn(n) = nx_block*ny_block + if (distribution_wght == 'block' .and. nocn(n) > 0) nocn(n) = nx_block*ny_block + if (.not. landblockelim) nocn(n) = max(nocn(n),1) #endif end do endif ! distribution_wght = file diff --git a/cicecore/cicedynB/infrastructure/ice_grid.F90 b/cicecore/cicedynB/infrastructure/ice_grid.F90 index 29853c96b..39750cc75 100644 --- a/cicecore/cicedynB/infrastructure/ice_grid.F90 +++ b/cicecore/cicedynB/infrastructure/ice_grid.F90 @@ -400,7 +400,7 @@ subroutine init_grid1 ! distribute blocks among processors !----------------------------------------------------------------- - call init_domain_distribution(work_g2, work_g1) ! KMT, ULAT + call init_domain_distribution(work_g2, work_g1, grid_ice) ! KMT, ULAT deallocate(work_g1) deallocate(work_g2) diff --git a/configuration/scripts/options/set_nml.histdbg b/configuration/scripts/options/set_nml.histdbg new file mode 100644 index 000000000..247d185fd --- /dev/null +++ b/configuration/scripts/options/set_nml.histdbg @@ -0,0 +1,225 @@ + histfreq = 'm','d','1','h','x' + histfreq_n = 1,1,1,1,1 + histfreq_base = 'zero' + write_ic = .true. + f_tmask = .true. + f_blkmask = .true. + f_tarea = .true. + f_uarea = .true. + f_dxt = .true. + f_dyt = .true. + f_dxu = .true. + f_dyu = .true. + f_HTN = .true. + f_HTE = .true. + f_ANGLE = .true. + f_ANGLET = .true. + f_NCAT = .true. + f_VGRDi = .true. + f_VGRDs = .true. + f_VGRDb = .true. + f_VGRDa = .true. + f_bounds = .true. + f_CMIP = 'm' + f_aice = 'md1h' + f_hi = 'h1dm' + f_hs = 'd1m' + f_Tsfc = 'mdh1' + f_sice = 'md' + f_uvel = 'md1' + f_vvel = 'dm1' + f_uatm = 'dm1' + f_vatm = 'dm1' + f_fswdn = 'dm1' + f_flwdn = 'md1' + f_snowfrac = 'md1' + f_snow = 'md1' + f_snow_ai = 'md1' + f_rain = 'md1' + f_rain_ai = 'md1' + f_sst = 'md1' + f_sss = 'md1' + f_uocn = 'md1' + f_vocn = 'md1' + f_frzmlt = 'md' + f_fswfac = 'md' + f_fswint_ai = 'md' + f_fswabs = 'md' + f_fswabs_ai = 'md' + f_albsni = 'md' + f_alvdr = 'md' + f_alidr = 'md' + f_alvdf = 'md' + f_alidf = 'md' + f_alvdr_ai = 'md' + f_alidr_ai = 'md' + f_alvdf_ai = 'md' + f_alidf_ai = 'md' + f_albice = 'md' + f_albsno = 'md' + f_albpnd = 'md' + f_coszen = 'md' + f_flat = 'md' + f_flat_ai = 'md' + f_fsens = 'md' + f_fsens_ai = 'md' + f_fswup = 'md' + f_flwup = 'md' + f_flwup_ai = 'md' + f_evap = 'md' + f_evap_ai = 'md' + f_Tair = 'md' + f_Tref = 'md' + f_Qref = 'md' + f_congel = 'md' + f_frazil = 'md' + f_snoice = 'md' + f_dsnow = 'md' + f_melts = 'md' + f_meltt = 'md' + f_meltb = 'md' + f_meltl = 'md' + f_fresh = 'md' + f_fresh_ai = 'md' + f_fsalt = 'md' + f_fsalt_ai = 'md' + f_fbot = 'md' + f_fhocn = 'md' + f_fhocn_ai = 'md' + f_fswthru = 'md' + f_fswthru_ai = 'md' + f_fsurf_ai = 'md' + f_fcondtop_ai = 'md' + f_fmeltt_ai = 'md' + f_strairx = 'md1' + f_strairy = 'md1' + f_strtltx = 'md1' + f_strtlty = 'md1' + f_strcorx = 'md1' + f_strcory = 'md1' + f_strocnx = 'md1' + f_strocny = 'md1' + f_strintx = 'md1' + f_strinty = 'md1' + f_taubx = 'md1' + f_tauby = 'md1' + f_strength = 'md1' + f_divu = 'md1' + f_shear = 'md1' + f_sig1 = 'md1' + f_sig2 = 'md1' + f_sigP = 'md1' + f_dvidtt = 'md' + f_dvidtd = 'md' + f_daidtt = 'md' + f_daidtd = 'md' + f_dagedtt = 'md' + f_dagedtd = 'md' + f_mlt_onset = 'md' + f_frz_onset = 'md' + f_hisnap = 'md' + f_aisnap = 'md' + f_trsig = 'md' + f_icepresent = 'md' + f_iage = 'md' + f_FY = 'md' + f_aicen = 'md' + f_vicen = 'md' + f_vsnon = 'md' + f_snowfracn = 'md' + f_keffn_top = 'md' + f_Tinz = 'md' + f_Sinz = 'md' + f_Tsnz = 'md' + f_fsurfn_ai = 'md' + f_fcondtopn_ai = 'md' + f_fmelttn_ai = 'md' + f_flatn_ai = 'md' + f_fsensn_ai = 'md' + f_alvl = 'md' + f_vlvl = 'md' + f_ardg = 'md' + f_vrdg = 'md' + f_dardg1dt = 'md' + f_dardg2dt = 'md' + f_dvirdgdt = 'md' + f_opening = 'md' + f_ardgn = 'md' + f_vrdgn = 'md' + f_dardg1ndt = 'md' + f_dardg2ndt = 'md' + f_dvirdgndt = 'md' + f_krdgn = 'md' + f_aparticn = 'md' + f_aredistn = 'md' + f_vredistn = 'md' + f_araftn = 'md' + f_vraftn = 'md' + f_apondn = 'md' + f_apeffn = 'md' + f_hpondn = 'md' + f_apond = 'md' + f_hpond = 'md' + f_ipond = 'md' + f_apeff = 'md' + f_apond_ai = 'md' + f_hpond_ai = 'md' + f_ipond_ai = 'md' + f_apeff_ai = 'md' + f_fiso_atm = 'md' + f_fiso_ocn = 'md' + f_iso = 'md' + f_faero_atm = 'md' + f_faero_ocn = 'md' + f_aero = 'md' + f_fbio = 'md' + f_fbio_ai = 'md' + f_zaero = 'md' + f_bgc_S = 'md' + f_bgc_N = 'md' + f_bgc_C = 'md' + f_bgc_DOC = 'md' + f_bgc_DIC = 'md' + f_bgc_chl = 'md' + f_bgc_Nit = 'md' + f_bgc_Am = 'md' + f_bgc_Sil = 'md' + f_bgc_DMSPp = 'md' + f_bgc_DMSPd = 'md' + f_bgc_DMS = 'md' + f_bgc_DON = 'md' + f_bgc_Fe = 'md' + f_bgc_hum = 'md' + f_bgc_PON = 'md' + f_bgc_ml = 'md' + f_upNO = 'md' + f_upNH = 'md' + f_bTin = 'md' + f_bphi = 'md' + f_iDi = 'md' + f_iki = 'md' + f_fbri = 'md' + f_hbri = 'md' + f_zfswin = 'md' + f_bionet = 'md' + f_biosnow = 'md' + f_grownet = 'md' + f_PPnet = 'md' + f_algalpeak = 'md' + f_zbgc_frac = 'md' + f_drag = 'md' + f_Cdn_atm = 'md' + f_Cdn_ocn = 'md' + f_fsdrad = 'md' + f_fsdperim = 'md' + f_afsd = 'md' + f_afsdn = 'md' + f_dafsd_newi = 'md' + f_dafsd_latg = 'md' + f_dafsd_latm = 'md' + f_dafsd_wave = 'md' + f_dafsd_weld = 'md' + f_wave_sig_ht = 'md' + f_aice_ww = 'md' + f_diam_ww = 'md' + f_hice_ww = 'md' diff --git a/configuration/scripts/tests/gridsys_suite.ts b/configuration/scripts/tests/gridsys_suite.ts index 30b9d7183..aa3150be3 100644 --- a/configuration/scripts/tests/gridsys_suite.ts +++ b/configuration/scripts/tests/gridsys_suite.ts @@ -17,6 +17,14 @@ smoke gbox80 4x2 boxsymne,kmtislands smoke gbox80 8x1 boxislandsn smoke gbox80 4x2 boxislandse smoke gbox80 2x4 boxislandsne +smoke gx3 1x1x100x116x1 reprosum,run10day +smoke gx3 1x1x25x29x16 reprosum,run10day,dwblockall smoke_gx3_1x1x100x116x1_reprosum_run10day +smoke gx3 1x1x5x4x580 reprosum,run10day,dwblockall smoke_gx3_1x1x100x116x1_reprosum_run10day +smoke gx3 1x1x5x4x580 reprosum,run10day smoke_gx3_1x1x100x116x1_reprosum_run10day +smoke gx1 32x1x16x16x32 reprosum,run10day +smoke gx1 32x1x16x16x32 reprosum,run10day,cmplogrest,dwblockall smoke_gx1_32x1x16x16x32_reprosum_run10day +smoke gx1 32x1x16x12x40 reprosum,run10day,cmplogrest,dwblockall smoke_gx1_32x1x16x16x32_reprosum_run10day +smoke gx1 32x1x16x12x40 reprosum,run10day,cmplogrest smoke_gx1_32x1x16x16x32_reprosum_run10day smoke gx3 8x2 diag1,run5day,gridcd restart gx3 4x2 debug,diag1,gridcd @@ -36,6 +44,14 @@ smoke gbox80 4x2 boxsymne,kmtislands,gridcd smoke gbox80 8x1 boxislandsn,gridcd smoke gbox80 4x2 boxislandse,gridcd smoke gbox80 2x4 boxislandsne,gridcd +smoke gx3 1x1x100x116x1 reprosum,run10day,gridcd +smoke gx3 1x1x25x29x16 reprosum,run10day,dwblockall,gridcd smoke_gx3_1x1x100x116x1_gridcd_reprosum_run10day +smoke gx3 1x1x5x4x580 reprosum,run10day,dwblockall,gridcd smoke_gx3_1x1x100x116x1_gridcd_reprosum_run10day +smoke gx3 1x1x5x4x580 reprosum,run10day,gridcd smoke_gx3_1x1x100x116x1_gridcd_reprosum_run10day +smoke gx1 32x1x16x16x32 reprosum,run10day,gridcd +smoke gx1 32x1x16x16x32 reprosum,run10day,cmplogrest,dwblockall,gridcd smoke_gx1_32x1x16x16x32_gridcd_reprosum_run10day +smoke gx1 32x1x16x12x40 reprosum,run10day,cmplogrest,dwblockall,gridcd smoke_gx1_32x1x16x16x32_gridcd_reprosum_run10day +smoke gx1 32x1x16x12x40 reprosum,run10day,cmplogrest,gridcd smoke_gx1_32x1x16x16x32_gridcd_reprosum_run10day smoke gx3 8x2 diag1,run5day,gridc restart gx3 4x2 debug,diag1,gridc @@ -55,3 +71,11 @@ smoke gbox80 4x2 boxsymne,kmtislands,gridc smoke gbox80 8x1 boxislandsn,gridc smoke gbox80 4x2 boxislandse,gridc smoke gbox80 2x4 boxislandsne,gridc +smoke gx3 1x1x100x116x1 reprosum,run10day,gridc +smoke gx3 1x1x25x29x16 reprosum,run10day,dwblockall,gridc smoke_gx3_1x1x100x116x1_gridc_reprosum_run10day +smoke gx3 1x1x5x4x580 reprosum,run10day,dwblockall,gridc smoke_gx3_1x1x100x116x1_gridc_reprosum_run10day +smoke gx3 1x1x5x4x580 reprosum,run10day,gridc smoke_gx3_1x1x100x116x1_gridc_reprosum_run10day +smoke gx1 32x1x16x16x32 reprosum,run10day,gridc +smoke gx1 32x1x16x16x32 reprosum,run10day,cmplogrest,dwblockall,gridc smoke_gx1_32x1x16x16x32_gridc_reprosum_run10day +smoke gx1 32x1x16x12x40 reprosum,run10day,cmplogrest,dwblockall,gridc smoke_gx1_32x1x16x16x32_gridc_reprosum_run10day +smoke gx1 32x1x16x12x40 reprosum,run10day,cmplogrest,gridc smoke_gx1_32x1x16x16x32_gridc_reprosum_run10day diff --git a/configuration/scripts/tests/omp_suite.ts b/configuration/scripts/tests/omp_suite.ts index d814e2b3d..fdf27881a 100644 --- a/configuration/scripts/tests/omp_suite.ts +++ b/configuration/scripts/tests/omp_suite.ts @@ -15,6 +15,7 @@ smoke gx3 11x2 isotope,reprosum,run10day smoke gx3 8x4 snwitdrdg,snwgrain,icdefault,reprosum,run10day smoke gx3 6x4 dynpicard,reprosum,run10day smoke gx3 8x3 zsal,reprosum,run10day +smoke gx3 1x1x100x116x1 reprosum,run10day,thread smoke gbox128 8x2 reprosum,run10day smoke gbox128 12x2 boxnodyn,reprosum,run10day @@ -39,6 +40,8 @@ smoke gx3 8x1 isotope,reprosum,run10day,cmplogrest,thread smoke gx3 8x1 snwitdrdg,snwgrain,icdefault,reprosum,run10day,cmplogrest,thread smoke_gx3_8x4_icdefault_reprosum_run10day_snwitdrdg_snwgrain smoke gx3 8x1 dynpicard,reprosum,run10day,cmplogrest,thread smoke_gx3_6x4_dynpicard_reprosum_run10day smoke gx3 8x1 zsal,reprosum,run10day,cmplogrest,thread smoke_gx3_8x3_reprosum_run10day_zsal +smoke gx3 4x2x25x29x4 reprosum,run10day smoke_gx3_1x1x100x116x1_reprosum_run10day_thread +smoke gx3 8x4x5x4x80 reprosum,run10day smoke_gx3_1x1x100x116x1_reprosum_run10day_thread smoke gbox128 8x1 reprosum,run10day,cmplogrest,thread smoke_gbox128_8x2_reprosum_run10day smoke gbox128 8x1 boxnodyn,reprosum,run10day,cmplogrest,thread smoke_gbox128_12x2_boxnodyn_reprosum_run10day @@ -64,6 +67,7 @@ smoke gx3 11x2 isotope,reprosum,run10day,gridc smoke gx3 8x4 snwitdrdg,snwgrain,icdefault,reprosum,run10day,gridc smoke gx3 6x4 dynpicard,reprosum,run10day,gridc smoke gx3 8x3 zsal,reprosum,run10day,gridc +smoke gx3 1x1x100x116x1 reprosum,run10day,gridc,thread smoke gbox128 8x2 reprosum,run10day,gridc smoke gbox128 12x2 boxnodyn,reprosum,run10day,gridc @@ -88,6 +92,8 @@ smoke gx3 8x1 isotope,reprosum,run10day,cmplogrest,thread,gr smoke gx3 8x1 snwitdrdg,snwgrain,icdefault,reprosum,run10day,cmplogrest,thread,gridc smoke_gx3_8x4_gridc_icdefault_reprosum_run10day_snwitdrdg_snwgrain smoke gx3 8x1 dynpicard,reprosum,run10day,cmplogrest,thread,gridc smoke_gx3_6x4_dynpicard_gridc_reprosum_run10day smoke gx3 8x1 zsal,reprosum,run10day,cmplogrest,thread,gridc smoke_gx3_8x3_gridc_reprosum_run10day_zsal +smoke gx3 4x2x25x29x4 reprosum,run10day,gridc smoke_gx3_1x1x100x116x1_gridc_reprosum_run10day_thread +smoke gx3 8x4x5x4x80 reprosum,run10day,gridc smoke_gx3_1x1x100x116x1_gridc_reprosum_run10day_thread smoke gbox128 8x1 reprosum,run10day,cmplogrest,thread,gridc smoke_gbox128_8x2_gridc_reprosum_run10day smoke gbox128 8x1 boxnodyn,reprosum,run10day,cmplogrest,thread,gridc smoke_gbox128_12x2_boxnodyn_gridc_reprosum_run10day @@ -113,6 +119,7 @@ smoke gx3 11x2 isotope,reprosum,run10day,gridcd smoke gx3 8x4 snwitdrdg,snwgrain,icdefault,reprosum,run10day,gridcd smoke gx3 6x4 dynpicard,reprosum,run10day,gridcd smoke gx3 8x3 zsal,reprosum,run10day,gridcd +smoke gx3 1x1x100x116x1 reprosum,run10day,gridcd,thread smoke gbox128 8x2 reprosum,run10day,gridcd smoke gbox128 12x2 boxnodyn,reprosum,run10day,gridcd @@ -137,6 +144,8 @@ smoke gx3 8x1 isotope,reprosum,run10day,cmplogrest,thread,gr smoke gx3 8x1 snwitdrdg,snwgrain,icdefault,reprosum,run10day,cmplogrest,thread,gridcd smoke_gx3_8x4_gridcd_icdefault_reprosum_run10day_snwitdrdg_snwgrain smoke gx3 8x1 dynpicard,reprosum,run10day,cmplogrest,thread,gridcd smoke_gx3_6x4_dynpicard_gridcd_reprosum_run10day smoke gx3 8x1 zsal,reprosum,run10day,cmplogrest,thread,gridcd smoke_gx3_8x3_gridcd_reprosum_run10day_zsal +smoke gx3 4x2x25x29x4 reprosum,run10day,gridcd smoke_gx3_1x1x100x116x1_gridcd_reprosum_run10day_thread +smoke gx3 8x4x5x4x80 reprosum,run10day,gridcd smoke_gx3_1x1x100x116x1_gridcd_reprosum_run10day_thread smoke gbox128 8x1 reprosum,run10day,cmplogrest,thread,gridcd smoke_gbox128_8x2_gridcd_reprosum_run10day smoke gbox128 8x1 boxnodyn,reprosum,run10day,cmplogrest,thread,gridcd smoke_gbox128_12x2_boxnodyn_gridcd_reprosum_run10day From c5d3a71e0fdaa3ae8a35554e0f0b627f97614ca2 Mon Sep 17 00:00:00 2001 From: apcraig Date: Mon, 14 Mar 2022 15:22:11 -0600 Subject: [PATCH 2/2] - Fix query_field error, fixes C/CD restart error - Add 2 day restart test, restart2 --- .../infrastructure/ice_read_write.F90 | 2 +- .../infrastructure/ice_restart_driver.F90 | 45 +++++----- .../io/io_netcdf/ice_restart.F90 | 11 +-- .../infrastructure/io/io_pio2/ice_restart.F90 | 13 ++- .../scripts/options/test_nml.restart21 | 7 ++ .../scripts/options/test_nml.restart22 | 6 ++ configuration/scripts/tests/gridsys_suite.ts | 3 + .../scripts/tests/test_restart2.files | 2 + .../scripts/tests/test_restart2.script | 82 +++++++++++++++++++ 9 files changed, 135 insertions(+), 36 deletions(-) create mode 100644 configuration/scripts/options/test_nml.restart21 create mode 100644 configuration/scripts/options/test_nml.restart22 create mode 100644 configuration/scripts/tests/test_restart2.files create mode 100644 configuration/scripts/tests/test_restart2.script diff --git a/cicecore/cicedynB/infrastructure/ice_read_write.F90 b/cicecore/cicedynB/infrastructure/ice_read_write.F90 index bf0361cf1..2443d75a3 100644 --- a/cicecore/cicedynB/infrastructure/ice_read_write.F90 +++ b/cicecore/cicedynB/infrastructure/ice_read_write.F90 @@ -1670,7 +1670,7 @@ subroutine ice_read_nc_xyf(fid, nrec, varname, work, diag, & amin = minval(work_g1(:,:,n)) amax = maxval(work_g1(:,:,n), mask = work_g1(:,:,n) /= missingvalue) asum = sum (work_g1(:,:,n), mask = work_g1(:,:,n) /= missingvalue) - write(nu_diag,*) subname,' min, max, sum =', amin, amax, asum + write(nu_diag,*) subname,' min, max, sum =', amin, amax, asum, trim(varname) enddo endif diff --git a/cicecore/cicedynB/infrastructure/ice_restart_driver.F90 b/cicecore/cicedynB/infrastructure/ice_restart_driver.F90 index 978ca7f55..07f05af30 100644 --- a/cicecore/cicedynB/infrastructure/ice_restart_driver.F90 +++ b/cicecore/cicedynB/infrastructure/ice_restart_driver.F90 @@ -19,6 +19,7 @@ module ice_restart_driver use ice_kinds_mod use ice_arrays_column, only: oceanmixed_ice + use ice_communicate, only: my_task, master_task use ice_constants, only: c0, c1, p5, & field_loc_center, field_loc_NEcorner, & field_loc_Eface, field_loc_Nface, & @@ -222,29 +223,29 @@ subroutine dumpfile(filename_spec) if (grid_ice == 'CD' .or. grid_ice == 'C') then - !$OMP PARALLEL DO PRIVATE(iblk,i,j) - do iblk = 1, nblocks - do j = 1, ny_block - do i = 1, nx_block - work1(i,j,iblk) = c0 - if (icenmask(i,j,iblk)) work1(i,j,iblk) = c1 - enddo - enddo - enddo - !$OMP END PARALLEL DO - call write_restart_field(nu_dump,0,work1,'ruf8','icenmask',1,diag) - - !$OMP PARALLEL DO PRIVATE(iblk,i,j) - do iblk = 1, nblocks - do j = 1, ny_block - do i = 1, nx_block - work1(i,j,iblk) = c0 - if (iceemask(i,j,iblk)) work1(i,j,iblk) = c1 + !$OMP PARALLEL DO PRIVATE(iblk,i,j) + do iblk = 1, nblocks + do j = 1, ny_block + do i = 1, nx_block + work1(i,j,iblk) = c0 + if (icenmask(i,j,iblk)) work1(i,j,iblk) = c1 + enddo + enddo enddo + !$OMP END PARALLEL DO + call write_restart_field(nu_dump,0,work1,'ruf8','icenmask',1,diag) + + !$OMP PARALLEL DO PRIVATE(iblk,i,j) + do iblk = 1, nblocks + do j = 1, ny_block + do i = 1, nx_block + work1(i,j,iblk) = c0 + if (iceemask(i,j,iblk)) work1(i,j,iblk) = c1 + enddo + enddo enddo - enddo - !$OMP END PARALLEL DO - call write_restart_field(nu_dump,0,work1,'ruf8','iceemask',1,diag) + !$OMP END PARALLEL DO + call write_restart_field(nu_dump,0,work1,'ruf8','iceemask',1,diag) endif @@ -266,7 +267,6 @@ subroutine restartfile (ice_ic) use ice_boundary, only: ice_HaloUpdate_stress use ice_blocks, only: nghost, nx_block, ny_block use ice_calendar, only: istep0, npt, calendar - use ice_communicate, only: my_task, master_task use ice_domain, only: nblocks, halo_info use ice_domain_size, only: nilyr, nslyr, ncat, & max_blocks @@ -697,7 +697,6 @@ subroutine restartfile_v4 (ice_ic) use ice_blocks, only: nghost, nx_block, ny_block use ice_calendar, only: istep0, istep1, timesecs, calendar, npt, & set_date_from_timesecs - use ice_communicate, only: my_task, master_task use ice_domain, only: nblocks, distrb_info use ice_domain_size, only: nilyr, nslyr, ncat, nx_global, ny_global, & max_blocks diff --git a/cicecore/cicedynB/infrastructure/io/io_netcdf/ice_restart.F90 b/cicecore/cicedynB/infrastructure/io/io_netcdf/ice_restart.F90 index d49764375..f117384d9 100644 --- a/cicecore/cicedynB/infrastructure/io/io_netcdf/ice_restart.F90 +++ b/cicecore/cicedynB/infrastructure/io/io_netcdf/ice_restart.F90 @@ -10,6 +10,7 @@ module ice_restart use ice_broadcast + use ice_communicate, only: my_task, master_task use ice_kinds_mod #ifdef USE_NETCDF use netcdf @@ -45,7 +46,6 @@ subroutine init_restart_read(ice_ic) use ice_calendar, only: msec, mmonth, mday, myear, & istep0, istep1, npt - use ice_communicate, only: my_task, master_task character(len=char_len_long), intent(in), optional :: ice_ic @@ -132,7 +132,6 @@ subroutine init_restart_write(filename_spec) use ice_blocks, only: nghost use ice_calendar, only: msec, mmonth, mday, myear, istep1 - use ice_communicate, only: my_task, master_task use ice_domain_size, only: nx_global, ny_global, ncat, nilyr, nslyr, & n_iso, n_aero, nblyr, n_zaero, n_algae, n_doc, & n_dic, n_don, n_fed, n_fep, nfsd @@ -841,7 +840,6 @@ end subroutine write_restart_field subroutine final_restart() use ice_calendar, only: istep1, idate - use ice_communicate, only: my_task, master_task integer (kind=int_kind) :: status @@ -904,8 +902,11 @@ logical function query_field(nu,vname) query_field = .false. #ifdef USE_NETCDF - status = nf90_inq_varid(ncid,trim(vname),varid) - if (status == nf90_noerr) query_field = .true. + if (my_task == master_task) then + status = nf90_inq_varid(ncid,trim(vname),varid) + if (status == nf90_noerr) query_field = .true. + endif + call broadcast_scalar(query_field,master_task) #else call abort_ice(subname//'ERROR: USE_NETCDF cpp not defined for '//trim(ice_ic), & file=__FILE__, line=__LINE__) diff --git a/cicecore/cicedynB/infrastructure/io/io_pio2/ice_restart.F90 b/cicecore/cicedynB/infrastructure/io/io_pio2/ice_restart.F90 index ce3946db2..e585788b7 100644 --- a/cicecore/cicedynB/infrastructure/io/io_pio2/ice_restart.F90 +++ b/cicecore/cicedynB/infrastructure/io/io_pio2/ice_restart.F90 @@ -6,6 +6,7 @@ module ice_restart use ice_broadcast + use ice_communicate, only: my_task, master_task use ice_exit, only: abort_ice use ice_fileunits, only: nu_diag, nu_restart, nu_rst_pointer use ice_kinds_mod @@ -44,7 +45,6 @@ subroutine init_restart_read(ice_ic) use ice_calendar, only: istep0, istep1, myear, mmonth, & mday, msec, npt - use ice_communicate, only: my_task, master_task use ice_domain_size, only: ncat use ice_read_write, only: ice_open @@ -140,7 +140,6 @@ end subroutine init_restart_read subroutine init_restart_write(filename_spec) use ice_calendar, only: msec, mmonth, mday, myear, istep1 - use ice_communicate, only: my_task, master_task use ice_domain_size, only: nx_global, ny_global, ncat, nilyr, nslyr, & n_iso, n_aero, nblyr, n_zaero, n_algae, n_doc, & n_dic, n_don, n_fed, n_fep, nfsd @@ -696,7 +695,6 @@ subroutine read_restart_field(nu,nrec,work,atype,vname,ndim3,diag, & field_loc, field_type) use ice_blocks, only: nx_block, ny_block - use ice_communicate, only: my_task, master_task use ice_constants, only: c0, field_loc_center use ice_boundary, only: ice_HaloUpdate use ice_domain, only: halo_info, distrb_info, nblocks @@ -815,7 +813,6 @@ end subroutine read_restart_field subroutine write_restart_field(nu,nrec,work,atype,vname,ndim3,diag) use ice_blocks, only: nx_block, ny_block - use ice_communicate, only: my_task, master_task use ice_constants, only: c0, field_loc_center use ice_domain, only: distrb_info, nblocks use ice_domain_size, only: max_blocks, ncat @@ -901,7 +898,6 @@ end subroutine write_restart_field subroutine final_restart() use ice_calendar, only: istep1, idate, msec - use ice_communicate, only: my_task, master_task character(len=*), parameter :: subname = '(final_restart)' @@ -951,8 +947,11 @@ logical function query_field(nu,vname) query_field = .false. #ifdef USE_NETCDF - status = pio_inq_varid(File,trim(vname),vardesc) - if (status == PIO_noerr) query_field = .true. + if (my_task == master_task) then + status = pio_inq_varid(File,trim(vname),vardesc) + if (status == PIO_noerr) query_field = .true. + endif + call broadcast_scalar(query_field,master_task) #else call abort_ice(subname//'ERROR: USE_NETCDF cpp not defined for '//trim(ice_ic), & file=__FILE__, line=__LINE__) diff --git a/configuration/scripts/options/test_nml.restart21 b/configuration/scripts/options/test_nml.restart21 new file mode 100644 index 000000000..7e4281ff6 --- /dev/null +++ b/configuration/scripts/options/test_nml.restart21 @@ -0,0 +1,7 @@ +npt = 2 +npt_unit = 'd' +dumpfreq = 'd' +dumpfreq_n = 1 +dumpfreq_base = 'init' +runtype = 'initial' +use_restart_time = .false. diff --git a/configuration/scripts/options/test_nml.restart22 b/configuration/scripts/options/test_nml.restart22 new file mode 100644 index 000000000..edc3e975a --- /dev/null +++ b/configuration/scripts/options/test_nml.restart22 @@ -0,0 +1,6 @@ +npt = 2 +npt_unit = 'd' +dumpfreq = 'd' +dumpfreq_n = 2 +dumpfreq_base = 'init' +runtype = 'continue' diff --git a/configuration/scripts/tests/gridsys_suite.ts b/configuration/scripts/tests/gridsys_suite.ts index aa3150be3..01132ff83 100644 --- a/configuration/scripts/tests/gridsys_suite.ts +++ b/configuration/scripts/tests/gridsys_suite.ts @@ -1,6 +1,7 @@ # Test Grid PEs Sets BFB-compare smoke gx3 8x2 diag1,run5day restart gx3 4x2 debug,diag1 +restart2 gx1 16x2 debug,diag1 smoke gbox12 1x1x12x12x1 boxchan smoke gbox80 1x1 box2001 smoke gbox80 2x2 boxwallp5 @@ -28,6 +29,7 @@ smoke gx1 32x1x16x12x40 reprosum,run10day,cmplogrest smoke gx3 8x2 diag1,run5day,gridcd restart gx3 4x2 debug,diag1,gridcd +restart2 gx1 16x2 debug,diag1,gridcd smoke gbox12 1x1x12x12x1 boxchan,gridcd smoke gbox80 1x1 box2001,gridcd smoke gbox80 2x2 boxwallp5,gridcd @@ -55,6 +57,7 @@ smoke gx1 32x1x16x12x40 reprosum,run10day,cmplogrest,gridcd smoke gx3 8x2 diag1,run5day,gridc restart gx3 4x2 debug,diag1,gridc +restart2 gx1 16x2 debug,diag1,gridc smoke gbox12 1x1x12x12x1 boxchan,gridc smoke gbox80 1x1 box2001,gridc smoke gbox80 2x2 boxwallp5,gridc diff --git a/configuration/scripts/tests/test_restart2.files b/configuration/scripts/tests/test_restart2.files new file mode 100644 index 000000000..7c22abe3a --- /dev/null +++ b/configuration/scripts/tests/test_restart2.files @@ -0,0 +1,2 @@ +test_nml.restart21 +test_nml.restart22 diff --git a/configuration/scripts/tests/test_restart2.script b/configuration/scripts/tests/test_restart2.script new file mode 100644 index 000000000..67760bbf4 --- /dev/null +++ b/configuration/scripts/tests/test_restart2.script @@ -0,0 +1,82 @@ + +# Build around a 2 day run with restart at day 1. +#----------------------------------------------------------- +# Run the CICE model baseline simulation + +cp ice_in ice_in.0 +${ICE_CASEDIR}/casescripts/parse_namelist.sh ice_in ${ICE_CASEDIR}/casescripts/test_nml.restart21 +cp ice_in ice_in.1 + +./cice.run +set res="$status" + +if ( $res != 0 ) then + mv -f ${ICE_CASEDIR}/test_output ${ICE_CASEDIR}/test_output.prev + cat ${ICE_CASEDIR}/test_output.prev | grep -iv "${ICE_TESTNAME} run" >! ${ICE_CASEDIR}/test_output + mv -f ${ICE_CASEDIR}/test_output ${ICE_CASEDIR}/test_output.prev + cat ${ICE_CASEDIR}/test_output.prev | grep -iv "${ICE_TESTNAME} test " >! ${ICE_CASEDIR}/test_output + rm -f ${ICE_CASEDIR}/test_output.prev + echo "FAIL ${ICE_TESTNAME} run" >> ${ICE_CASEDIR}/test_output + echo "FAIL ${ICE_TESTNAME} test " >> ${ICE_CASEDIR}/test_output + exit 99 +endif + +# Prepend 'base_' to the final restart file to save for comparison +if ( "${ICE_IOTYPE}" == "binary" ) then + set end_date = `ls -t1 ${ICE_RUNDIR}/restart | head -1 | awk -F'.' '{print $NF}'` + foreach file (${ICE_RUNDIR}/restart/*${end_date}) + set surname = `echo $file | awk -F'/' '{print $NF}'` + mv $file ${ICE_RUNDIR}/restart/base_$surname + end +else + set test_file = `ls -t1 ${ICE_RUNDIR}/restart | head -1` + set test_data = ${ICE_RUNDIR}/restart/${test_file} + set base_data = ${ICE_RUNDIR}/restart/base_${test_file} + mv ${test_data} ${base_data} +endif + +#----------------------------------------------------------- +# Run the CICE model for the restart simulation + +# Modify the contents of the pointer file for restart +perl -i -pe's/(\d{4})-(\d{2})-(\d{2})/sprintf("%04d-%02d-%02d",$1,$2,$3-1)/e' ${ICE_RUNDIR}/ice.restart_file + +${ICE_CASEDIR}/casescripts/parse_namelist.sh ice_in ${ICE_CASEDIR}/casescripts/test_nml.restart22 +cp ice_in ice_in.2 + +./cice.run +set res="$status" + +cp ice_in.0 ice_in + +mv -f ${ICE_CASEDIR}/test_output ${ICE_CASEDIR}/test_output.prev +cat ${ICE_CASEDIR}/test_output.prev | grep -iv "${ICE_TESTNAME} run" >! ${ICE_CASEDIR}/test_output +mv -f ${ICE_CASEDIR}/test_output ${ICE_CASEDIR}/test_output.prev +cat ${ICE_CASEDIR}/test_output.prev | grep -iv "${ICE_TESTNAME} test" >! ${ICE_CASEDIR}/test_output +rm -f ${ICE_CASEDIR}/test_output.prev + +if ( $res != 0 ) then + echo "FAIL ${ICE_TESTNAME} run " >> ${ICE_CASEDIR}/test_output + echo "FAIL ${ICE_TESTNAME} test " >> ${ICE_CASEDIR}/test_output + exit 99 +else + set log_file = `ls -t1 ${ICE_RUNDIR}/cice.runlog* | head -1` + set ttimeloop = `grep TimeLoop ${log_file} | grep Timer | cut -c 22-32` + set tdynamics = `grep Dynamics ${log_file} | grep Timer | cut -c 22-32` + set tcolumn = `grep Column ${log_file} | grep Timer | cut -c 22-32` + if (${ttimeloop} == "") set ttimeloop = -1 + if (${tdynamics} == "") set tdynamics = -1 + if (${tcolumn} == "") set tcolumn = -1 + echo "PASS ${ICE_TESTNAME} run ${ttimeloop} ${tdynamics} ${tcolumn}" >> ${ICE_CASEDIR}/test_output + + ${ICE_CASEDIR}/casescripts/comparebfb.csh ${ICE_RUNDIR}/restart + set bfbstatus = $status + if (${bfbstatus} == 0) then + echo "PASS ${ICE_TESTNAME} test " >> ${ICE_CASEDIR}/test_output + else + echo "FAIL ${ICE_TESTNAME} test " >> ${ICE_CASEDIR}/test_output + endif +endif + +#----------------------------------------------------------- +