From 27426fdd12aee877a241836c96af2e841ebd6e03 Mon Sep 17 00:00:00 2001 From: Jian Sun Date: Wed, 15 Jan 2025 10:37:21 -0700 Subject: [PATCH 1/6] change clubb openacc directives to passpass GPU tests --- src/physics/cam/clubb_intr.F90 | 8 ++++---- 1 file changed, 4 insertions(+), 4 deletions(-) diff --git a/src/physics/cam/clubb_intr.F90 b/src/physics/cam/clubb_intr.F90 index a9f25f0256..79def73767 100644 --- a/src/physics/cam/clubb_intr.F90 +++ b/src/physics/cam/clubb_intr.F90 @@ -2869,10 +2869,13 @@ subroutine clubb_tend_cam( state, ptend_all, pbuf, hdtime, & !$acc rrho, prer_evap, rtp2_mc_zt, thlp2_mc_zt, wprtp_mc_zt, wpthlp_mc_zt, rtpthlp_mc_zt ) & !$acc copy( um, vm, upwp, vpwp, wpthvp, wp2thvp, rtpthvp, thlpthvp, up2, vp2, up3, vp3, & !$acc wp2, wp3, rtp2, thlp2, rtp3, thlp3, thlm, rtm, rvm, wprtp, wpthlp, rtpthlp, & + !$acc pdf_zm_w_1, pdf_zm_w_2, pdf_zm_varnce_w_1, pdf_zm_varnce_w_2, pdf_zm_mixt_frac, & !$acc cloud_frac, wp2rtp, wp2thlp, uprcp, vprcp, rc_coef, wp4, wpup2, wpvp2, & !$acc ttend_clubb_mc, upwp_clubb_gw_mc, vpwp_clubb_gw_mc, thlp2_clubb_gw_mc, wpthlp_clubb_gw_mc, & !$acc ttend_clubb, upwp_clubb_gw, vpwp_clubb_gw, thlp2_clubb_gw, wpthlp_clubb_gw, & - !$acc wp2up2, wp2vp2, ice_supersat_frac ) & + !$acc wp2up2, wp2vp2, ice_supersat_frac, & + !$acc pdf_params_zm_chnk(lchnk)%w_1, pdf_params_zm_chnk(lchnk)%w_2, & + !$acc pdf_params_zm_chnk(lchnk)%varnce_w_1, pdf_params_zm_chnk(lchnk)%varnce_w_2 ) & !$acc copyout( temp2d, temp2dp, rtp2_zt_out, thl2_zt_out, wp2_zt_out, pdfp_rtp2, wm_zt_out, inv_exner_clubb, & !$acc rcm, wprcp, rcm_in_layer, cloud_cover, zt_out, zi_out, khzm, qclvar, thv, dz_g, & !$acc clubbtop, se_dis, eleak, clubb_s, wpthvp_clubb, wprcp_clubb ) & @@ -2893,7 +2896,6 @@ subroutine clubb_tend_cam( state, ptend_all, pbuf, hdtime, & !$acc radf, wpthlp_sfc, clubb_params, sfc_elevation, wprtp_sfc, upwp_sfc, vpwp_sfc, & !$acc rtm_ref, thlm_ref, um_ref, vm_ref, ug, vg, p_in_Pa, exner, um_pert_inout, & !$acc inv_exner_clubb_surf, thlprcp_out, zi_g, zt_g, qrl_clubb, & - !$acc pdf_zm_w_1, pdf_zm_w_2, pdf_zm_varnce_w_1, pdf_zm_varnce_w_2, pdf_zm_mixt_frac, & !$acc pdf_params_chnk(lchnk)%w_1, pdf_params_chnk(lchnk)%w_2, & !$acc pdf_params_chnk(lchnk)%varnce_w_1, pdf_params_chnk(lchnk)%varnce_w_2, & !$acc pdf_params_chnk(lchnk)%rt_1, pdf_params_chnk(lchnk)%rt_2, & @@ -2917,8 +2919,6 @@ subroutine clubb_tend_cam( state, ptend_all, pbuf, hdtime, & !$acc pdf_params_chnk(lchnk)%cloud_frac_1, pdf_params_chnk(lchnk)%cloud_frac_2, & !$acc pdf_params_chnk(lchnk)%mixt_frac, pdf_params_chnk(lchnk)%ice_supersat_frac_1, & !$acc pdf_params_chnk(lchnk)%ice_supersat_frac_2, & - !$acc pdf_params_zm_chnk(lchnk)%w_1, pdf_params_zm_chnk(lchnk)%w_2, & - !$acc pdf_params_zm_chnk(lchnk)%varnce_w_1, pdf_params_zm_chnk(lchnk)%varnce_w_2, & !$acc pdf_params_zm_chnk(lchnk)%rt_1, pdf_params_zm_chnk(lchnk)%rt_2, & !$acc pdf_params_zm_chnk(lchnk)%varnce_rt_1, pdf_params_zm_chnk(lchnk)%varnce_rt_2, & !$acc pdf_params_zm_chnk(lchnk)%thl_1, pdf_params_zm_chnk(lchnk)%thl_2, & From 881197746538517adc7c686a2e45e69945ad1fe9 Mon Sep 17 00:00:00 2001 From: Jian Sun Date: Wed, 15 Jan 2025 11:23:56 -0700 Subject: [PATCH 2/6] reminder for a new ccs_config tag --- .gitmodules | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/.gitmodules b/.gitmodules index 752c3bf4f7..78c604aa6f 100644 --- a/.gitmodules +++ b/.gitmodules @@ -144,7 +144,7 @@ fxDONOTUSEurl = https://github.com/ESCOMP/mizuRoute [submodule "ccs_config"] path = ccs_config url = https://github.com/ESMCI/ccs_config_cesm.git -fxtag = ccs_config_cesm1.0.8 +fxtag = I_need_a_new_tag_here fxrequired = ToplevelRequired fxDONOTUSEurl = https://github.com/ESMCI/ccs_config_cesm.git From b3a012aab549ca9e9682d19502a66cd067ff890e Mon Sep 17 00:00:00 2001 From: Jian Sun Date: Tue, 21 Jan 2025 08:57:28 -0700 Subject: [PATCH 3/6] update ccs_config tag for clubb gpu fix --- .gitmodules | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/.gitmodules b/.gitmodules index 78c604aa6f..c57c1a8235 100644 --- a/.gitmodules +++ b/.gitmodules @@ -144,7 +144,7 @@ fxDONOTUSEurl = https://github.com/ESCOMP/mizuRoute [submodule "ccs_config"] path = ccs_config url = https://github.com/ESMCI/ccs_config_cesm.git -fxtag = I_need_a_new_tag_here +fxtag = ccs_config_cesm1.0.21 fxrequired = ToplevelRequired fxDONOTUSEurl = https://github.com/ESMCI/ccs_config_cesm.git From 7495bf5250410f76ed5a046f16ef8a5b0c4c89f1 Mon Sep 17 00:00:00 2001 From: Jian Sun Date: Tue, 21 Jan 2025 16:20:46 -0700 Subject: [PATCH 4/6] add Gunther's suggestion --- src/physics/cam/clubb_intr.F90 | 6 +++--- 1 file changed, 3 insertions(+), 3 deletions(-) diff --git a/src/physics/cam/clubb_intr.F90 b/src/physics/cam/clubb_intr.F90 index 79def73767..c99c7f1c9a 100644 --- a/src/physics/cam/clubb_intr.F90 +++ b/src/physics/cam/clubb_intr.F90 @@ -2875,7 +2875,8 @@ subroutine clubb_tend_cam( state, ptend_all, pbuf, hdtime, & !$acc ttend_clubb, upwp_clubb_gw, vpwp_clubb_gw, thlp2_clubb_gw, wpthlp_clubb_gw, & !$acc wp2up2, wp2vp2, ice_supersat_frac, & !$acc pdf_params_zm_chnk(lchnk)%w_1, pdf_params_zm_chnk(lchnk)%w_2, & - !$acc pdf_params_zm_chnk(lchnk)%varnce_w_1, pdf_params_zm_chnk(lchnk)%varnce_w_2 ) & + !$acc pdf_params_zm_chnk(lchnk)%varnce_w_1, pdf_params_zm_chnk(lchnk)%varnce_w_2, & + !$acc pdf_params_zm_chnk(lchnk)%mixt_frac ) & !$acc copyout( temp2d, temp2dp, rtp2_zt_out, thl2_zt_out, wp2_zt_out, pdfp_rtp2, wm_zt_out, inv_exner_clubb, & !$acc rcm, wprcp, rcm_in_layer, cloud_cover, zt_out, zi_out, khzm, qclvar, thv, dz_g, & !$acc clubbtop, se_dis, eleak, clubb_s, wpthvp_clubb, wprcp_clubb ) & @@ -2938,8 +2939,7 @@ subroutine clubb_tend_cam( state, ptend_all, pbuf, hdtime, & !$acc pdf_params_zm_chnk(lchnk)%corr_chi_eta_2, pdf_params_zm_chnk(lchnk)%rsatl_1, & !$acc pdf_params_zm_chnk(lchnk)%rsatl_2, pdf_params_zm_chnk(lchnk)%rc_1, pdf_params_zm_chnk(lchnk)%rc_2, & !$acc pdf_params_zm_chnk(lchnk)%cloud_frac_1, pdf_params_zm_chnk(lchnk)%cloud_frac_2, & - !$acc pdf_params_zm_chnk(lchnk)%mixt_frac, pdf_params_zm_chnk(lchnk)%ice_supersat_frac_1, & - !$acc pdf_params_zm_chnk(lchnk)%ice_supersat_frac_2 ) + !$acc pdf_params_zm_chnk(lchnk)%ice_supersat_frac_1, pdf_params_zm_chnk(lchnk)%ice_supersat_frac_2 ) !$acc data if( sclr_dim > 0 ) & !$acc create( wpsclrp_sfc, sclrm_forcing, sclrm, wpsclrp, sclrp2, sclrp3, sclrprtp, sclrpthlp, sclrpthvp_inout) & From 2bf795418acafc3efa44630a54c97c31fab03e7d Mon Sep 17 00:00:00 2001 From: Jesse Nusbaumer Date: Mon, 10 Feb 2025 12:08:49 -0700 Subject: [PATCH 5/6] Initial ChangeLog entry, including Derecho test results. --- ccs_config | 2 +- doc/ChangeLog | 92 +++++++++++++++++++++++++++++++++++++++++++++++++++ 2 files changed, 93 insertions(+), 1 deletion(-) diff --git a/ccs_config b/ccs_config index 775e9f7900..f3cae52b30 160000 --- a/ccs_config +++ b/ccs_config @@ -1 +1 @@ -Subproject commit 775e9f790044c3632e70e2beda9d66db34558b7b +Subproject commit f3cae52b3096639d767778fa8033efe5e2d79cf0 diff --git a/doc/ChangeLog b/doc/ChangeLog index 570d28ad94..acf9c06b37 100644 --- a/doc/ChangeLog +++ b/doc/ChangeLog @@ -1,4 +1,96 @@ +=============================================================== + +Tag name: cam6_4_064 +Originator(s): sjsprecious, huebleruwm +Date: Feb 10, 2025 +One-line Summary: Fix broken GPU tests for CLUBB code +Github PR URL: https://github.com/ESCOMP/CAM/pull/1226 + +Purpose of changes (include the issue number and title text for each relevant +GitHub issue): + +This PR fixes the broken ERS tests due to the recent GPU changes of CLUBB code +(PR #1175). + +Note that this PR need a new ccs_config tag from ESMCI/ccs_config_cesm#204 to +complete this PR. + +Closes #1220 - GPU test fails restart comparison + +Describe any changes made to build system: N/A +Describe any changes made to the namelist: N/A + +List any changes to the defaults for the boundary datasets: N?A + +Describe any substantial timing or memory changes: N/A + +Code reviewed by: nusbaume + +List all files eliminated: N/A + +List all files added and what they do: N/A + +List all existing files that have been modified, and describe the changes: + +M .gitmodules + - Update ccs_config to ccs_config_cesm1.0.21 (needed for test fix) + +M src/physics/cam/clubb_intr.F90 + - Move variables from OpenACC create to copy call to fix restart test. + +If there were any failures reported from running test_driver.sh on any test +platform, and checkin with these failures has been OK'd by the gatekeeper, +then copy the lines from the td.*.status files for the failed tests to the +appropriate machine below. All failed tests must be justified. + +derecho/intel/aux_cam: + +ERP_Ln9.f09_f09_mg17.FCSD_HCO.derecho_intel.cam-outfrq9s (Overall: FAIL) +SMS_Ld1.f09_f09_mg17.FCHIST_GC.derecho_intel.cam-outfrq1d (Overall: DIFF) + - pre-existing failure due to HEMCO not having reproducible results issues #1018 and #856 + +SMS_D_Ln9.f19_f19_mg17.FXHIST.derecho_intel.cam-outfrq9s_amie (Overall: FAIL) +SMS_D_Ln9_P1280x1.ne0CONUSne30x8_ne0CONUSne30x8_mt12.FCHIST.derecho_intel.cam-outfrq9s (Overall: FAIL) + - pre-existing failures due to build-namelist error requiring CLM/CTSM external update + +SMS_D_Ln9_P1280x1.ne30pg3_ne30pg3_mg17.FHISTC_MTt1s.derecho_intel.cam-outfrq9s_Leung_dust (Overall: DIFF) + - Missed baseline update from previous CAM tag (cam6_4_063) + +derecho/nvhpc/aux_cam: + +ERS_Ln9.ne30pg3_ne30pg3_mg17.F2000dev.derecho_nvhpc.cam-outfrq9s_gpu_default (Overall: DIFF) + - Expected change in baseline answers. + +izumi/nag/aux_cam: + +izumi/gnu/aux_cam: + +CAM tag used for the baseline comparison tests if different than previous +tag: + +Summarize any changes to answers, i.e., +- what code configurations: +- what platforms/compilers: +- nature of change (roundoff; larger than roundoff but same climate; new + climate): + +If bitwise differences were observed, how did you show they were no worse +than roundoff? + +If this tag changes climate describe the run(s) done to evaluate the new +climate in enough detail that it(they) could be reproduced, i.e., +- source tag (all code used must be in the repository): +- platform/compilers: +- configure commandline: +- build-namelist command (or complete namelist): +- MSS location of output: + +MSS location of control simulations used to validate new climate: + +URL for AMWG diagnostics output used to validate new climate: + +=============================================================== =============================================================== Tag name: cam6_4_063 From cbdaefdd600205269ec6ac3fd2437b2510ef077e Mon Sep 17 00:00:00 2001 From: Jesse Nusbaumer Date: Mon, 10 Feb 2025 13:25:05 -0700 Subject: [PATCH 6/6] Finalize ChangeLog with Izumi test results. --- doc/ChangeLog | 12 ++++++------ 1 file changed, 6 insertions(+), 6 deletions(-) diff --git a/doc/ChangeLog b/doc/ChangeLog index acf9c06b37..0aaa217c2a 100644 --- a/doc/ChangeLog +++ b/doc/ChangeLog @@ -21,7 +21,7 @@ Describe any changes made to build system: N/A Describe any changes made to the namelist: N/A -List any changes to the defaults for the boundary datasets: N?A +List any changes to the defaults for the boundary datasets: N/A Describe any substantial timing or memory changes: N/A @@ -34,10 +34,10 @@ List all files added and what they do: N/A List all existing files that have been modified, and describe the changes: M .gitmodules - - Update ccs_config to ccs_config_cesm1.0.21 (needed for test fix) + - Update ccs_config submodule to ccs_config_cesm1.0.21 (needed for test fix) M src/physics/cam/clubb_intr.F90 - - Move variables from OpenACC create to copy call to fix restart test. + - Move variables from OpenACC create call to copy call to fix restart test. If there were any failures reported from running test_driver.sh on any test platform, and checkin with these failures has been OK'd by the gatekeeper, @@ -48,7 +48,7 @@ derecho/intel/aux_cam: ERP_Ln9.f09_f09_mg17.FCSD_HCO.derecho_intel.cam-outfrq9s (Overall: FAIL) SMS_Ld1.f09_f09_mg17.FCHIST_GC.derecho_intel.cam-outfrq1d (Overall: DIFF) - - pre-existing failure due to HEMCO not having reproducible results issues #1018 and #856 + - pre-existing failure due to HEMCO not having reproducible results (issues #1018 and #856) SMS_D_Ln9.f19_f19_mg17.FXHIST.derecho_intel.cam-outfrq9s_amie (Overall: FAIL) SMS_D_Ln9_P1280x1.ne0CONUSne30x8_ne0CONUSne30x8_mt12.FCHIST.derecho_intel.cam-outfrq9s (Overall: FAIL) @@ -62,9 +62,9 @@ derecho/nvhpc/aux_cam: ERS_Ln9.ne30pg3_ne30pg3_mg17.F2000dev.derecho_nvhpc.cam-outfrq9s_gpu_default (Overall: DIFF) - Expected change in baseline answers. -izumi/nag/aux_cam: +izumi/nag/aux_cam: ALL PASS -izumi/gnu/aux_cam: +izumi/gnu/aux_cam: ALL PASS CAM tag used for the baseline comparison tests if different than previous tag: