From a19b8991201257cff92ee8f0ec861c37331ad388 Mon Sep 17 00:00:00 2001 From: Ufuk Turuncoglu Date: Thu, 23 May 2024 14:25:50 -0500 Subject: [PATCH 1/7] distribute land and ocean points evenly across processors --- drivers/nuopc/lnd_comp_domain.F90 | 189 +++++++++++++++++++++++++++++- 1 file changed, 188 insertions(+), 1 deletion(-) diff --git a/drivers/nuopc/lnd_comp_domain.F90 b/drivers/nuopc/lnd_comp_domain.F90 index db6bd8b..db62d45 100644 --- a/drivers/nuopc/lnd_comp_domain.F90 +++ b/drivers/nuopc/lnd_comp_domain.F90 @@ -23,6 +23,7 @@ module lnd_comp_domain use ESMF , only : ESMF_RouteHandleDestroy, ESMF_GridGet, ESMF_GridGetCoord use ESMF , only : ESMF_FieldRegridGetArea, ESMF_CoordSys_Flag use ESMF , only : ESMF_MeshGetFieldBounds, ESMF_COORDSYS_CART, ESMF_KIND_R8 + use ESMF , only : ESMF_MeshDestroy, ESMF_DistGridCreate, ESMF_VMAllGatherV use NUOPC, only : NUOPC_CompAttributeGet use lnd_comp_kind , only : r4 => shr_kind_r4 @@ -62,11 +63,29 @@ subroutine lnd_set_decomp_and_domain_from_mosaic(gcomp, noahmp, rc) ! local variables real(r4), target, allocatable :: tmpr4(:) real(r8), target, allocatable :: tmpr8(:) - integer :: n + integer :: n, m, g integer :: decomptile(2,6) integer :: maxIndex(2) type(ESMF_Decomp_Flag) :: decompflagPTile(2,6) + integer :: petCount, localPet + integer :: gsize(1), lsize, mytask + integer :: nlnd, nocn + integer, allocatable :: nlnd_loc(:), nocn_loc(:) + integer, pointer :: tmp(:), lsize_arr(:) + integer, pointer :: mask_glb(:) + integer, pointer :: gindex_loc(:) + integer, pointer :: gindex_glb(:) + integer, pointer :: gindex_lnd(:) + integer, pointer :: gindex_ocn(:) + integer, pointer :: gindex_new(:) + integer :: begl_l, endl_l + integer :: begl_o, endl_o + type(ESMF_VM) :: vm + type(ESMF_DistGrid) :: distgrid + type(ESMF_DistGrid) :: distgrid_new + type(ESMF_Mesh) :: mesh + type(field_type) :: flds(1) integer :: numOwnedElements, spatialDim, rank @@ -87,6 +106,16 @@ subroutine lnd_set_decomp_and_domain_from_mosaic(gcomp, noahmp, rc) rc = ESMF_SUCCESS call ESMF_LogWrite(subname//' called', ESMF_LOGMSG_INFO) + ! --------------------- + ! Query VM + ! --------------------- + + call ESMF_GridCompGet(gcomp, vm=vm, rc=rc) + if (ChkErr(rc,__LINE__,u_FILE_u)) return + + call ESMF_VMGet(vm, petCount=petCount, localPet=localPet, rc=rc) + if (ChkErr(rc,__LINE__,u_FILE_u)) return + ! --------------------- ! Set decomposition and decide it is regional or global ! --------------------- @@ -257,6 +286,164 @@ subroutine lnd_set_decomp_and_domain_from_mosaic(gcomp, noahmp, rc) call ESMF_MeshSet(noahmp%domain%mesh, elementMask=noahmp%domain%mask, rc=rc) if (chkerr(rc,__LINE__,u_FILE_u)) return + ! --------------------- + ! Modify decomposition + ! --------------------- + + ! retrive default distgrid + call ESMF_MeshGet(noahmp%domain%mesh, elementdistGrid=distgrid, rc=rc) + if (chkerr(rc,__LINE__,u_FILE_u)) return + + ! get local number of elements + call ESMF_DistGridGet(distgrid, localDe=0, elementCount=lsize, rc=rc) + if (chkerr(rc,__LINE__,u_FILE_u)) return + + ! calculate number of elements globally + call ESMF_VMAllReduce(vm, (/ lsize /), gsize, 1, ESMF_REDUCE_SUM, rc=rc) + if (chkerr(rc,__LINE__,u_FILE_u)) return + + nlnd = count(noahmp%domain%mask(:) > 0, dim=1) + nocn = lsize-nlnd + write(msg, fmt='(A,4I8)') trim(subname)//' : lsize, gsize, nlnd, nocn = ', & + lsize, gsize(1), nlnd, nocn + call ESMF_LogWrite(trim(msg), ESMF_LOGMSG_INFO) + + ! collect local sizes + allocate(lsize_arr(petCount)) + call ESMF_VMAllGatherV(vm, sendData=(/ lsize /), sendCount=1, & + recvData=lsize_arr, recvCounts=(/ (1, n = 0, petCount-1) /), & + recvOffsets=(/ (n, n = 0, petCount-1) /), rc=rc) + if (chkerr(rc,__LINE__,u_FILE_u)) return + + !do n = 1, petCount + ! write(msg,'(A,2I8)') trim(subname)//' : lsize = ', n, lsize_arr(n) + ! call ESMF_LogWrite(trim(msg), ESMF_LOGMSG_INFO) + !end do + + ! get default sequence index + allocate(gindex_loc(lsize)) + call ESMF_DistGridGet(distgrid, 0, seqIndexList=gindex_loc, rc=rc) + if (chkerr(rc,__LINE__,u_FILE_u)) return + + !write(msg,'(A,4I8)') trim(subname)//' : distgrid list = ',& + ! gindex_loc(1), gindex_loc(lsize), minval(gindex_loc), maxval(gindex_loc) + !call ESMF_LogWrite(trim(msg), ESMF_LOGMSG_INFO) + + ! create global index by collecting local indexes + allocate(gindex_glb(gsize(1))) + gindex_glb(:) = 0 + call ESMF_VMAllGatherV(vm, sendData=gindex_loc, sendCount=lsize, & + recvData=gindex_glb, recvCounts=lsize_arr, & + recvOffsets=(/ (sum(lsize_arr(1:n))-lsize_arr(1), n = 1, petCount) /), rc=rc) + if (chkerr(rc,__LINE__,u_FILE_u)) return + + ! generate global land sea mask + allocate(mask_glb(gsize(1))) + mask_glb(:) = 0 + call ESMF_VMAllGatherV(vm, sendData=noahmp%domain%mask, sendCount=lsize, & + recvData=mask_glb, recvCounts=lsize_arr, & + recvOffsets=(/ (sum(lsize_arr(1:n))-lsize_arr(1), n = 1, petCount) /), rc=rc) + if (chkerr(rc,__LINE__,u_FILE_u)) return + + ! split global indexes as land and ocean + nlnd = count(mask_glb > 0, dim=1) + nocn = gsize(1)-nlnd + allocate(gindex_lnd(nlnd)) + gindex_lnd = 0 + allocate(gindex_ocn(nocn)) + gindex_ocn = 0 + + n = 0 + m = 0 + do g = 1, gsize(1) + if (mask_glb(g) > 0) then + n = n+1 + gindex_lnd(n) = gindex_glb(g) + else + m = m+1 + gindex_ocn(m) = gindex_glb(g) + end if + end do + + ! create new local indexes + allocate(nlnd_loc(0:petCount-1)) + nlnd_loc = 0 + allocate(nocn_loc(0:petCount-1)) + nocn_loc = 0 + do n = 0, petCount-1 + nlnd_loc(n) = nlnd/petCount + nocn_loc(n) = nocn/petCount + if (n < mod(nlnd, petCount)) then + nlnd_loc(n) = nlnd_loc(n)+1 + else + nocn_loc(n) = nocn_loc(n)+1 + end if + end do + if (localPet == 0) then + begl_l = 1 + begl_o = 1 + else + begl_l = sum(nlnd_loc(0:localPet-1))+1 + begl_o = sum(nocn_loc(0:localPet-1))+1 + end if + endl_l = sum(nlnd_loc(0:localPet)) + endl_o = sum(nocn_loc(0:localPet)) + + write(msg,'(A,6I8)') trim(subname)//' : nlnd_loc, nocn_loc, begl_l, endl_l, begl_o, endl_o = ', nlnd_loc(localPet), nocn_loc(localPet), begl_l, endl_l, begl_o, endl_o + call ESMF_LogWrite(trim(msg), ESMF_LOGMSG_INFO) + + allocate(gindex_new(nlnd_loc(localPet)+nocn_loc(localPet))) + gindex_new(:nlnd_loc(localPet)) = gindex_lnd(begl_l:begl_l) + gindex_new(nlnd_loc(localPet)+1:) = gindex_ocn(begl_o:endl_o) + + do n = 1, nlnd_loc(localPet)+nocn_loc(localPet) + write(msg,'(A,2I8)') trim(subname)//' : n, gindex_new = ', n, gindex_new(n) + call ESMF_LogWrite(trim(msg), ESMF_LOGMSG_INFO) + end do + + ! update internal data structures + noahmp%domain%mask(:nlnd_loc(localPet)) = 1 + noahmp%domain%mask(nlnd_loc(localPet)+1:) = 0 + + ! read field + filename = trim(noahmp%nmlist%input_dir)//'oro_data.tile*.nc' + flds(1)%short_name = 'land_frac' + flds(1)%ptr1r4 => tmpr4 + call read_tiled_file(noahmp, filename, flds, rc=rc) + if (ChkErr(rc,__LINE__,u_FILE_u)) return + + ! allocate data + if (.not. allocated(noahmp%domain%frac)) then + allocate(noahmp%domain%frac(noahmp%domain%begl:noahmp%domain%endl)) + end if + noahmp%domain%frac = dble(tmpr4) + + ! create new distgrid with new index + distgrid_new = ESMF_DistGridCreate(arbSeqIndexList=gindex_new, rc=rc) + if (chkerr(rc,__LINE__,u_FILE_u)) return + + ! create new mesh with new distgrid + mesh = ESMF_MeshCreate(noahmp%domain%mesh, elementDistGrid=distgrid_new, rc=rc) + if (chkerr(rc,__LINE__,u_FILE_u)) return + + ! destroy old and replace with new one + call ESMF_MeshDestroy(noahmp%domain%mesh, rc=rc) + if (chkerr(rc,__LINE__,u_FILE_u)) return + noahmp%domain%mesh = mesh + + !call ESMF_MeshGetFieldBounds(noahmp%domain%mesh, meshloc=ESMF_MESHLOC_ELEMENT, & + ! totalLBound=tlb, totalUBound=tub, totalCount=tc, rc=rc) + !if (ChkErr(rc,__LINE__,u_FILE_u)) return + + !noahmp%domain%begl = tlb(1) + !noahmp%domain%endl = tub(1) + !noahmp%static%im = tc(1) + !write(msg, fmt='(A,3I8)') trim(subname)//' : begl, endl, im = ', noahmp%domain%begl, & + ! noahmp%domain%endl, noahmp%static%im + !call ESMF_LogWrite(trim(msg), ESMF_LOGMSG_INFO) + + !noahmp%domain%mask(noahmp%domain%begl:noahmp%domain%endl) = mask_glb(gindex_new) + ! --------------------- ! Get height from orography file ! --------------------- From 35e6bd033ea53e4666b49ddeb7e5ff48f20de945 Mon Sep 17 00:00:00 2001 From: Ufuk Turuncoglu Date: Fri, 24 May 2024 17:42:45 -0500 Subject: [PATCH 2/7] clean implementation --- drivers/nuopc/lnd_comp_domain.F90 | 342 ++++++++++++++++++------------ drivers/nuopc/lnd_comp_shr.F90 | 47 +++- drivers/nuopc/lnd_comp_types.F90 | 2 + 3 files changed, 242 insertions(+), 149 deletions(-) diff --git a/drivers/nuopc/lnd_comp_domain.F90 b/drivers/nuopc/lnd_comp_domain.F90 index db62d45..8abe77e 100644 --- a/drivers/nuopc/lnd_comp_domain.F90 +++ b/drivers/nuopc/lnd_comp_domain.F90 @@ -63,28 +63,11 @@ subroutine lnd_set_decomp_and_domain_from_mosaic(gcomp, noahmp, rc) ! local variables real(r4), target, allocatable :: tmpr4(:) real(r8), target, allocatable :: tmpr8(:) - integer :: n, m, g + integer :: n, petCount, localPet integer :: decomptile(2,6) integer :: maxIndex(2) type(ESMF_Decomp_Flag) :: decompflagPTile(2,6) - - integer :: petCount, localPet - integer :: gsize(1), lsize, mytask - integer :: nlnd, nocn - integer, allocatable :: nlnd_loc(:), nocn_loc(:) - integer, pointer :: tmp(:), lsize_arr(:) - integer, pointer :: mask_glb(:) - integer, pointer :: gindex_loc(:) - integer, pointer :: gindex_glb(:) - integer, pointer :: gindex_lnd(:) - integer, pointer :: gindex_ocn(:) - integer, pointer :: gindex_new(:) - integer :: begl_l, endl_l - integer :: begl_o, endl_o type(ESMF_VM) :: vm - type(ESMF_DistGrid) :: distgrid - type(ESMF_DistGrid) :: distgrid_new - type(ESMF_Mesh) :: mesh type(field_type) :: flds(1) integer :: numOwnedElements, spatialDim, rank @@ -130,7 +113,14 @@ subroutine lnd_set_decomp_and_domain_from_mosaic(gcomp, noahmp, rc) ! set number of tiles noahmp%domain%ntiles = 6 - ! set decomposition + ! check user provided layout + if (petCount /= noahmp%domain%layout(1)*noahmp%domain%layout(2)*noahmp%domain%ntiles) then + call ESMF_LogWrite(trim(subname)//": ERROR in layout. layout_x * layout_y * 6 != #PETs", ESMF_LOGMSG_INFO) + rc = ESMF_FAILURE + return + end if + + ! use user provided layout to set decomposition do n = 1, noahmp%domain%ntiles decomptile(1,n) = noahmp%domain%layout(1) decomptile(2,n) = noahmp%domain%layout(2) @@ -287,7 +277,144 @@ subroutine lnd_set_decomp_and_domain_from_mosaic(gcomp, noahmp, rc) if (chkerr(rc,__LINE__,u_FILE_u)) return ! --------------------- - ! Modify decomposition + ! Modify decomposition to evenly distribute land and ocean points + ! --------------------- + + if (trim(noahmp%nmlist%decomp_type) == 'custom') then + ! modify decomposition + call lnd_modify_decomp(gcomp, noahmp, rc) + if (chkerr(rc,__LINE__,u_FILE_u)) return + end if + + ! --------------------- + ! Get height from orography file + ! --------------------- + + ! read field + filename = trim(noahmp%nmlist%input_dir)//'oro_data.tile*.nc' + flds(1)%short_name = 'orog_raw' + flds(1)%ptr1r4 => tmpr4 + call read_tiled_file(noahmp, filename, flds, rc=rc) + if (ChkErr(rc,__LINE__,u_FILE_u)) return + + ! allocate data + if (.not. allocated(noahmp%domain%hgt)) then + allocate(noahmp%domain%hgt(noahmp%domain%begl:noahmp%domain%endl)) + end if + noahmp%domain%hgt = dble(tmpr4) + + ! --------------------- + ! Query cell area + ! --------------------- + + ! create field in R8 type + farea = ESMF_FieldCreate(noahmp%domain%mesh, ESMF_TYPEKIND_R8, & + meshloc=ESMF_MESHLOC_ELEMENT, rc=rc) + + ! get cell area to the field + call ESMF_FieldRegridGetArea(farea, rc=rc) + if (ChkErr(rc,__LINE__,u_FILE_u)) return + if (.not. allocated(noahmp%domain%garea)) then + allocate(noahmp%domain%garea(noahmp%domain%begl:noahmp%domain%endl)) + end if + + ! retrieve pointer and fill area array + call ESMF_FieldGet(farea, farrayPtr=ptr1d, rc=rc) + if (chkerr(rc,__LINE__,u_FILE_u)) return + noahmp%domain%garea(:) = ptr1d(:) + + ! make unit conversion from square radians to square meters if it is required + call ESMF_MeshGet(noahmp%domain%mesh, coordSys=coordSys, rc=rc) + if (chkerr(rc,__LINE__,u_FILE_u)) return + if (coordSys /= ESMF_COORDSYS_CART) then + noahmp%domain%garea(:) = noahmp%domain%garea(:)*(con_rerth**2) + end if + + ! --------------------- + ! Query coordiates from ESMF mesh + ! --------------------- + + ! determine dimensions in mesh + call ESMF_MeshGet(noahmp%domain%mesh, spatialDim=spatialDim, & + numOwnedElements=numOwnedElements, rc=rc) + if (chkerr(rc,__LINE__,u_FILE_u)) return + + ! obtain mesh longitudes and latitudes + if (.not. allocated(noahmp%domain%lats)) then + allocate(noahmp%domain%lats(numOwnedElements)) + end if + if (.not. allocated(noahmp%domain%lons)) then + allocate(noahmp%domain%lons(numOwnedElements)) + end if + if (.not. allocated(ownedElemCoords)) then + allocate(ownedElemCoords(spatialDim*numOwnedElements)) + end if + call ESMF_MeshGet(noahmp%domain%mesh, ownedElemCoords=ownedElemCoords) + if (chkerr(rc,__LINE__,u_FILE_u)) return + do n = 1,numOwnedElements + noahmp%domain%lons(n) = ownedElemCoords(2*n-1) + noahmp%domain%lats(n) = ownedElemCoords(2*n) + end do + deallocate(ownedElemCoords) + + ! --------------------- + ! Clean memory + ! --------------------- + + if (allocated(tmpr4)) deallocate(tmpr4) + if (allocated(tmpr8)) deallocate(tmpr8) + + call ESMF_LogWrite(subname//' done', ESMF_LOGMSG_INFO) + + end subroutine lnd_set_decomp_and_domain_from_mosaic + + !=============================================================================== + subroutine lnd_modify_decomp(gcomp, noahmp, rc) + + ! input/output variables + type(ESMF_GridComp), intent(in) :: gcomp + type(noahmp_type), intent(inout) :: noahmp + integer, intent(out) :: rc + + ! local variables + type(ESMF_VM) :: vm + type(ESMF_Mesh) :: mesh + type(ESMF_DistGrid) :: distgrid, distgrid_new + type(field_type) :: flds(1) + integer :: n, m, g + integer :: petCount, localPet + integer :: lsize, gsize(1) + integer :: nlnd, nocn + integer :: begl_l, endl_l, begl_o, endl_o + integer, allocatable :: nlnd_loc(:) + integer, allocatable :: nocn_loc(:) + integer, allocatable :: mask_glb(:) + integer, allocatable :: gindex_loc(:) + integer, allocatable :: gindex_glb(:) + integer, allocatable :: gindex_lnd(:) + integer, allocatable :: gindex_ocn(:) + integer, allocatable :: gindex_new(:) + integer, allocatable :: lsize_arr(:) + real(r4), target, allocatable :: tmpr4(:) + character(len=cl) :: msg, filename + character(len=*), parameter :: subname = trim(modName)//':(lnd_modify_decomp) ' + !------------------------------------------------------------------------------- + + rc = ESMF_SUCCESS + call ESMF_LogWrite(subname//' called', ESMF_LOGMSG_INFO) + + ! --------------------- + ! Query VM + ! --------------------- + + call ESMF_GridCompGet(gcomp, vm=vm, rc=rc) + if (ChkErr(rc,__LINE__,u_FILE_u)) return + + call ESMF_VMGet(vm, petCount=petCount, localPet=localPet, rc=rc) + if (ChkErr(rc,__LINE__,u_FILE_u)) return + + ! --------------------- + ! Query existing mesh ! --------------------- ! retrive default distgrid @@ -305,9 +432,18 @@ subroutine lnd_set_decomp_and_domain_from_mosaic(gcomp, noahmp, rc) nlnd = count(noahmp%domain%mask(:) > 0, dim=1) nocn = lsize-nlnd write(msg, fmt='(A,4I8)') trim(subname)//' : lsize, gsize, nlnd, nocn = ', & - lsize, gsize(1), nlnd, nocn + lsize, gsize(1), nlnd, nocn call ESMF_LogWrite(trim(msg), ESMF_LOGMSG_INFO) + ! get default sequence index + allocate(gindex_loc(lsize)) + call ESMF_DistGridGet(distgrid, 0, seqIndexList=gindex_loc, rc=rc) + if (chkerr(rc,__LINE__,u_FILE_u)) return + + ! --------------------- + ! Create global view of indexes and mask + ! -------------------- + ! collect local sizes allocate(lsize_arr(petCount)) call ESMF_VMAllGatherV(vm, sendData=(/ lsize /), sendCount=1, & @@ -315,21 +451,7 @@ subroutine lnd_set_decomp_and_domain_from_mosaic(gcomp, noahmp, rc) recvOffsets=(/ (n, n = 0, petCount-1) /), rc=rc) if (chkerr(rc,__LINE__,u_FILE_u)) return - !do n = 1, petCount - ! write(msg,'(A,2I8)') trim(subname)//' : lsize = ', n, lsize_arr(n) - ! call ESMF_LogWrite(trim(msg), ESMF_LOGMSG_INFO) - !end do - - ! get default sequence index - allocate(gindex_loc(lsize)) - call ESMF_DistGridGet(distgrid, 0, seqIndexList=gindex_loc, rc=rc) - if (chkerr(rc,__LINE__,u_FILE_u)) return - - !write(msg,'(A,4I8)') trim(subname)//' : distgrid list = ',& - ! gindex_loc(1), gindex_loc(lsize), minval(gindex_loc), maxval(gindex_loc) - !call ESMF_LogWrite(trim(msg), ESMF_LOGMSG_INFO) - - ! create global index by collecting local indexes + ! global view of indexes allocate(gindex_glb(gsize(1))) gindex_glb(:) = 0 call ESMF_VMAllGatherV(vm, sendData=gindex_loc, sendCount=lsize, & @@ -337,7 +459,7 @@ subroutine lnd_set_decomp_and_domain_from_mosaic(gcomp, noahmp, rc) recvOffsets=(/ (sum(lsize_arr(1:n))-lsize_arr(1), n = 1, petCount) /), rc=rc) if (chkerr(rc,__LINE__,u_FILE_u)) return - ! generate global land sea mask + ! global view of land sea mask allocate(mask_glb(gsize(1))) mask_glb(:) = 0 call ESMF_VMAllGatherV(vm, sendData=noahmp%domain%mask, sendCount=lsize, & @@ -345,7 +467,10 @@ subroutine lnd_set_decomp_and_domain_from_mosaic(gcomp, noahmp, rc) recvOffsets=(/ (sum(lsize_arr(1:n))-lsize_arr(1), n = 1, petCount) /), rc=rc) if (chkerr(rc,__LINE__,u_FILE_u)) return + ! --------------------- ! split global indexes as land and ocean + ! --------------------- + nlnd = count(mask_glb > 0, dim=1) nocn = gsize(1)-nlnd allocate(gindex_lnd(nlnd)) @@ -365,7 +490,10 @@ subroutine lnd_set_decomp_and_domain_from_mosaic(gcomp, noahmp, rc) end if end do + ! --------------------- ! create new local indexes + ! --------------------- + allocate(nlnd_loc(0:petCount-1)) nlnd_loc = 0 allocate(nocn_loc(0:petCount-1)) @@ -389,34 +517,29 @@ subroutine lnd_set_decomp_and_domain_from_mosaic(gcomp, noahmp, rc) endl_l = sum(nlnd_loc(0:localPet)) endl_o = sum(nocn_loc(0:localPet)) - write(msg,'(A,6I8)') trim(subname)//' : nlnd_loc, nocn_loc, begl_l, endl_l, begl_o, endl_o = ', nlnd_loc(localPet), nocn_loc(localPet), begl_l, endl_l, begl_o, endl_o - call ESMF_LogWrite(trim(msg), ESMF_LOGMSG_INFO) + if (noahmp%nmlist%debug_level > 10) then + write(msg,'(A,6I8)') trim(subname)//' : nlnd_loc, nocn_loc, begl_l, endl_l, begl_o, endl_o = ', nlnd_loc(localPet), nocn_loc(localPet), begl_l, endl_l, begl_o, endl_o + call ESMF_LogWrite(trim(msg), ESMF_LOGMSG_INFO) + end if allocate(gindex_new(nlnd_loc(localPet)+nocn_loc(localPet))) gindex_new(:nlnd_loc(localPet)) = gindex_lnd(begl_l:begl_l) gindex_new(nlnd_loc(localPet)+1:) = gindex_ocn(begl_o:endl_o) - do n = 1, nlnd_loc(localPet)+nocn_loc(localPet) - write(msg,'(A,2I8)') trim(subname)//' : n, gindex_new = ', n, gindex_new(n) - call ESMF_LogWrite(trim(msg), ESMF_LOGMSG_INFO) - end do - - ! update internal data structures - noahmp%domain%mask(:nlnd_loc(localPet)) = 1 - noahmp%domain%mask(nlnd_loc(localPet)+1:) = 0 + if (noahmp%nmlist%debug_level > 10) then + do n = 1, nlnd_loc(localPet)+nocn_loc(localPet) + write(msg,'(A,2I8)') trim(subname)//' : n, gindex_new = ', n, gindex_new(n) + call ESMF_LogWrite(trim(msg), ESMF_LOGMSG_INFO) + end do + end if - ! read field - filename = trim(noahmp%nmlist%input_dir)//'oro_data.tile*.nc' - flds(1)%short_name = 'land_frac' - flds(1)%ptr1r4 => tmpr4 - call read_tiled_file(noahmp, filename, flds, rc=rc) - if (ChkErr(rc,__LINE__,u_FILE_u)) return + write(msg, fmt='(A,4I8)') trim(subname)//' : lsize, gsize, nlnd, nocn = ', & + lsize, gsize(1), nlnd_loc(localPet), nocn_loc(localPet) + call ESMF_LogWrite(trim(msg), ESMF_LOGMSG_INFO) - ! allocate data - if (.not. allocated(noahmp%domain%frac)) then - allocate(noahmp%domain%frac(noahmp%domain%begl:noahmp%domain%endl)) - end if - noahmp%domain%frac = dble(tmpr4) + ! --------------------- + ! Update mesh with new decomposition + ! --------------------- ! create new distgrid with new index distgrid_new = ESMF_DistGridCreate(arbSeqIndexList=gindex_new, rc=rc) @@ -431,99 +554,40 @@ subroutine lnd_set_decomp_and_domain_from_mosaic(gcomp, noahmp, rc) if (chkerr(rc,__LINE__,u_FILE_u)) return noahmp%domain%mesh = mesh - !call ESMF_MeshGetFieldBounds(noahmp%domain%mesh, meshloc=ESMF_MESHLOC_ELEMENT, & - ! totalLBound=tlb, totalUBound=tub, totalCount=tc, rc=rc) - !if (ChkErr(rc,__LINE__,u_FILE_u)) return - - !noahmp%domain%begl = tlb(1) - !noahmp%domain%endl = tub(1) - !noahmp%static%im = tc(1) - !write(msg, fmt='(A,3I8)') trim(subname)//' : begl, endl, im = ', noahmp%domain%begl, & - ! noahmp%domain%endl, noahmp%static%im - !call ESMF_LogWrite(trim(msg), ESMF_LOGMSG_INFO) - - !noahmp%domain%mask(noahmp%domain%begl:noahmp%domain%endl) = mask_glb(gindex_new) - ! --------------------- - ! Get height from orography file + ! fix mask and fraction to be consistent with new decomposition ! --------------------- - ! read field + ! mask + noahmp%domain%mask(:nlnd_loc(localPet)) = 1 + noahmp%domain%mask(nlnd_loc(localPet)+1:) = 0 + + ! fraction, read from file again + allocate(tmpr4(noahmp%domain%begl:noahmp%domain%endl)) + tmpr4(:) = 0.0 filename = trim(noahmp%nmlist%input_dir)//'oro_data.tile*.nc' - flds(1)%short_name = 'orog_raw' + flds(1)%short_name = 'land_frac' flds(1)%ptr1r4 => tmpr4 call read_tiled_file(noahmp, filename, flds, rc=rc) if (ChkErr(rc,__LINE__,u_FILE_u)) return - - ! allocate data - if (.not. allocated(noahmp%domain%hgt)) then - allocate(noahmp%domain%hgt(noahmp%domain%begl:noahmp%domain%endl)) - end if - noahmp%domain%hgt = dble(tmpr4) - - ! --------------------- - ! Query cell area - ! --------------------- - - ! create field in R8 type - farea = ESMF_FieldCreate(noahmp%domain%mesh, ESMF_TYPEKIND_R8, & - meshloc=ESMF_MESHLOC_ELEMENT, rc=rc) - - ! get cell area to the field - call ESMF_FieldRegridGetArea(farea, rc=rc) - if (ChkErr(rc,__LINE__,u_FILE_u)) return - if (.not. allocated(noahmp%domain%garea)) then - allocate(noahmp%domain%garea(noahmp%domain%begl:noahmp%domain%endl)) - end if - - ! retrieve pointer and fill area array - call ESMF_FieldGet(farea, farrayPtr=ptr1d, rc=rc) - if (chkerr(rc,__LINE__,u_FILE_u)) return - noahmp%domain%garea(:) = ptr1d(:) - - ! make unit conversion from square radians to square meters if it is required - call ESMF_MeshGet(noahmp%domain%mesh, coordSys=coordSys, rc=rc) - if (chkerr(rc,__LINE__,u_FILE_u)) return - if (coordSys /= ESMF_COORDSYS_CART) then - noahmp%domain%garea(:) = noahmp%domain%garea(:)*(con_rerth**2) - end if - - ! --------------------- - ! Query coordiates from ESMF mesh - ! --------------------- - - ! determine dimensions in mesh - call ESMF_MeshGet(noahmp%domain%mesh, spatialDim=spatialDim, & - numOwnedElements=numOwnedElements, rc=rc) - if (chkerr(rc,__LINE__,u_FILE_u)) return - - ! obtain mesh longitudes and latitudes - if (.not. allocated(noahmp%domain%lats)) then - allocate(noahmp%domain%lats(numOwnedElements)) - end if - if (.not. allocated(noahmp%domain%lons)) then - allocate(noahmp%domain%lons(numOwnedElements)) - end if - if (.not. allocated(ownedElemCoords)) then - allocate(ownedElemCoords(spatialDim*numOwnedElements)) - end if - call ESMF_MeshGet(noahmp%domain%mesh, ownedElemCoords=ownedElemCoords) - if (chkerr(rc,__LINE__,u_FILE_u)) return - do n = 1,numOwnedElements - noahmp%domain%lons(n) = ownedElemCoords(2*n-1) - noahmp%domain%lats(n) = ownedElemCoords(2*n) - end do - deallocate(ownedElemCoords) + deallocate(tmpr4) ! --------------------- ! Clean memory - ! --------------------- - - if (allocated(tmpr4)) deallocate(tmpr4) - if (allocated(tmpr8)) deallocate(tmpr8) + ! --------------------- + + deallocate(nlnd_loc) + deallocate(nocn_loc) + deallocate(mask_glb) + deallocate(gindex_loc) + deallocate(gindex_glb) + deallocate(gindex_lnd) + deallocate(gindex_ocn) + deallocate(gindex_new) + deallocate(lsize_arr) call ESMF_LogWrite(subname//' done', ESMF_LOGMSG_INFO) - end subroutine lnd_set_decomp_and_domain_from_mosaic + end subroutine lnd_modify_decomp end module lnd_comp_domain diff --git a/drivers/nuopc/lnd_comp_shr.F90 b/drivers/nuopc/lnd_comp_shr.F90 index 6229568..f92b3c5 100644 --- a/drivers/nuopc/lnd_comp_shr.F90 +++ b/drivers/nuopc/lnd_comp_shr.F90 @@ -811,17 +811,17 @@ subroutine read_namelist(gcomp, noahmp, rc) if (chkerr(rc,__LINE__,u_FILE_u)) return if (isPresent .and. isSet) then read(cvalue,*) noahmp%nmlist%output_mode - if (trim(noahmp%nmlist%output_mode) == 'all' .or. & - trim(noahmp%nmlist%output_mode) == 'mid' .or. & - trim(noahmp%nmlist%output_mode) == 'low') then - else - call ESMF_LogWrite(trim(subname)//": ERROR in output_mode. Only 'all', 'mid' and 'low' are allowed!", ESMF_LOGMSG_INFO) - rc = ESMF_FAILURE - return + if (trim(noahmp%nmlist%output_mode) /= 'all' .and. & + trim(noahmp%nmlist%output_mode) /= 'mid' .and. & + trim(noahmp%nmlist%output_mode) /= 'low') then + call ESMF_LogWrite(trim(subname)//" : ERROR in output_mode. Only 'all', 'mid' and 'low' are allowed!", ESMF_LOGMSG_INFO) + rc = ESMF_FAILURE + return end if else noahmp%nmlist%output_mode = 'all' end if + call ESMF_LogWrite(trim(subname)//' : output_mode = '//trim(noahmp%nmlist%output_mode), ESMF_LOGMSG_INFO) ! restart frequency call NUOPC_CompAttributeGet(gcomp, name='restart_freq', value=cvalue, isPresent=isPresent, isSet=isSet, rc=rc) @@ -831,7 +831,7 @@ subroutine read_namelist(gcomp, noahmp, rc) else noahmp%nmlist%restart_freq = noahmp%nmlist%output_freq end if - write(msg, fmt='(A,I6)') trim(subname)//': restart_freq = ', noahmp%nmlist%restart_freq + write(msg, fmt='(A,I6)') trim(subname)//' : restart_freq = ', noahmp%nmlist%restart_freq call ESMF_LogWrite(trim(msg), ESMF_LOGMSG_INFO) ! MYNN-EDMF @@ -841,7 +841,7 @@ subroutine read_namelist(gcomp, noahmp, rc) if (isPresent .and. isSet) then if (trim(cvalue) .eq. '.true.' .or. trim(cvalue) .eq. 'true') noahmp%static%do_mynnedmf = .true. end if - write(msg, fmt='(A,L)') trim(subname)//': do_mynnedmf = ', noahmp%static%do_mynnedmf + write(msg, fmt='(A,L)') trim(subname)//' : do_mynnedmf = ', noahmp%static%do_mynnedmf call ESMF_LogWrite(trim(msg), ESMF_LOGMSG_INFO) ! MYNN Surface Layer Scheme @@ -853,7 +853,7 @@ subroutine read_namelist(gcomp, noahmp, rc) end if noahmp%model%do_mynnsfclay = noahmp%static%do_mynnsfclay if (noahmp%static%iopt_sfc == 4) noahmp%model%do_mynnsfclay = .true. - write(msg, fmt='(A,L)') trim(subname)//': do_mynnsfclay = ', noahmp%static%do_mynnsfclay + write(msg, fmt='(A,L)') trim(subname)//' : do_mynnsfclay = ', noahmp%static%do_mynnsfclay call ESMF_LogWrite(trim(msg), ESMF_LOGMSG_INFO) ! Option for soil type category @@ -902,6 +902,33 @@ subroutine read_namelist(gcomp, noahmp, rc) write(msg, fmt='(A,F6.2)') trim(subname)//' : initial_albedo = ', noahmp%nmlist%initial_albedo call ESMF_LogWrite(trim(msg), ESMF_LOGMSG_INFO) + ! Decomposition + call NUOPC_CompAttributeGet(gcomp, name='decomp_type', value=cvalue, isPresent=isPresent, isSet=isSet, rc=rc) + if (chkerr(rc,__LINE__,u_FILE_u)) return + if (isPresent .and. isSet) then + read(cvalue,*) noahmp%nmlist%decomp_type + if (trim(noahmp%nmlist%decomp_type) /= 'default' .and. & + trim(noahmp%nmlist%decomp_type) /= 'custom') then + call ESMF_LogWrite(trim(subname)//": ERROR in decomp_type. Only 'default' and 'custom' are allowed!", ESMF_LOGMSG_INFO) + rc = ESMF_FAILURE + return + end if + else + noahmp%nmlist%decomp_type = 'default' + end if + call ESMF_LogWrite(trim(subname)//' : decomp_type = '//trim(noahmp%nmlist%decomp_type), ESMF_LOGMSG_INFO) + + ! Debug level + call NUOPC_CompAttributeGet(gcomp, name='debug_level', value=cvalue, isPresent=isPresent, isSet=isSet, rc=rc) + if (chkerr(rc,__LINE__,u_FILE_u)) return + if (isPresent .and. isSet) then + read(cvalue,*) noahmp%nmlist%debug_level + else + noahmp%nmlist%debug_level = 0 + end if + write(msg, fmt='(A,I1)') trim(subname)//' : debug_level = ', noahmp%nmlist%debug_level + call ESMF_LogWrite(trim(msg), ESMF_LOGMSG_INFO) + call ESMF_LogWrite(subname//' done', ESMF_LOGMSG_INFO) end subroutine read_namelist diff --git a/drivers/nuopc/lnd_comp_types.F90 b/drivers/nuopc/lnd_comp_types.F90 index bec9fbd..888ee8f 100644 --- a/drivers/nuopc/lnd_comp_types.F90 +++ b/drivers/nuopc/lnd_comp_types.F90 @@ -278,6 +278,8 @@ module lnd_comp_types integer :: veg_type_category ! vegetation type (category) real(kind=r8) :: initial_emiss ! initial value for the emissivity (constant in everywhere) real(kind=r8) :: initial_albedo ! initial value for the monthly albedo (constant in everywhere) + character*10 :: decomp_type ! decomposition type: default or custom + logical :: debug_level ! debug level end type namelist_type type noahmp_type From 97e5dc7b48cc4b2f808229de66e2e12bce871c03 Mon Sep 17 00:00:00 2001 From: Ufuk Turuncoglu Date: Mon, 22 Jul 2024 16:20:35 -0500 Subject: [PATCH 3/7] minor fix for debug --- drivers/nuopc/lnd_comp_domain.F90 | 11 +++-------- 1 file changed, 3 insertions(+), 8 deletions(-) diff --git a/drivers/nuopc/lnd_comp_domain.F90 b/drivers/nuopc/lnd_comp_domain.F90 index 8abe77e..3299edf 100644 --- a/drivers/nuopc/lnd_comp_domain.F90 +++ b/drivers/nuopc/lnd_comp_domain.F90 @@ -517,10 +517,9 @@ subroutine lnd_modify_decomp(gcomp, noahmp, rc) endl_l = sum(nlnd_loc(0:localPet)) endl_o = sum(nocn_loc(0:localPet)) - if (noahmp%nmlist%debug_level > 10) then - write(msg,'(A,6I8)') trim(subname)//' : nlnd_loc, nocn_loc, begl_l, endl_l, begl_o, endl_o = ', nlnd_loc(localPet), nocn_loc(localPet), begl_l, endl_l, begl_o, endl_o - call ESMF_LogWrite(trim(msg), ESMF_LOGMSG_INFO) - end if + write(msg,'(A,6I8)') trim(subname)//' : nlnd_loc, nocn_loc, begl_l, endl_l, begl_o, endl_o = ', & + nlnd_loc(localPet), nocn_loc(localPet), begl_l, endl_l, begl_o, endl_o + call ESMF_LogWrite(trim(msg), ESMF_LOGMSG_INFO) allocate(gindex_new(nlnd_loc(localPet)+nocn_loc(localPet))) gindex_new(:nlnd_loc(localPet)) = gindex_lnd(begl_l:begl_l) @@ -533,10 +532,6 @@ subroutine lnd_modify_decomp(gcomp, noahmp, rc) end do end if - write(msg, fmt='(A,4I8)') trim(subname)//' : lsize, gsize, nlnd, nocn = ', & - lsize, gsize(1), nlnd_loc(localPet), nocn_loc(localPet) - call ESMF_LogWrite(trim(msg), ESMF_LOGMSG_INFO) - ! --------------------- ! Update mesh with new decomposition ! --------------------- From d765daf14706f3dbd8b27f0275c9b0b92c8e7271 Mon Sep 17 00:00:00 2001 From: Ufuk Turuncoglu Date: Fri, 2 Aug 2024 16:39:11 -0500 Subject: [PATCH 4/7] add option to specify fixed file directory --- drivers/nuopc/lnd_comp_domain.F90 | 2 +- drivers/nuopc/lnd_comp_io.F90 | 14 +++++++------- drivers/nuopc/lnd_comp_nuopc.F90 | 10 ++++++++++ drivers/nuopc/lnd_comp_types.F90 | 1 + 4 files changed, 19 insertions(+), 8 deletions(-) diff --git a/drivers/nuopc/lnd_comp_domain.F90 b/drivers/nuopc/lnd_comp_domain.F90 index 3299edf..56f1f97 100644 --- a/drivers/nuopc/lnd_comp_domain.F90 +++ b/drivers/nuopc/lnd_comp_domain.F90 @@ -245,7 +245,7 @@ subroutine lnd_set_decomp_and_domain_from_mosaic(gcomp, noahmp, rc) if (ChkErr(rc,__LINE__,u_FILE_u)) return vegtype(:) = int(tmpr8) else - write(filename, fmt="(A,I0,A)") trim(noahmp%nmlist%input_dir)//'C',noahmp%domain%ni, '.vegetation_type.tile*.nc' + write(filename, fmt="(A,I0,A)") trim(noahmp%nmlist%fixed_dir)//'C',noahmp%domain%ni, '.vegetation_type.tile*.nc' flds(1)%short_name = 'vegetation_type' flds(1)%ptr1r4 => tmpr4 call read_tiled_file(noahmp, filename, flds, rc=rc) diff --git a/drivers/nuopc/lnd_comp_io.F90 b/drivers/nuopc/lnd_comp_io.F90 index 2bd3299..eade850 100644 --- a/drivers/nuopc/lnd_comp_io.F90 +++ b/drivers/nuopc/lnd_comp_io.F90 @@ -532,7 +532,7 @@ subroutine read_static(noahmp, rc) deallocate(flds) else allocate(flds(1)) - write(filename, fmt="(A,I0,A)") trim(noahmp%nmlist%input_dir)//'C', noahmp%domain%ni, '.soil_type.tile*.nc' + write(filename, fmt="(A,I0,A)") trim(noahmp%nmlist%fixed_dir)//'C', noahmp%domain%ni, '.soil_type.tile*.nc' flds(1)%short_name = 'soil_type' flds(1)%ptr1r4 => tmpr4 call read_tiled_file(noahmp, filename, flds, rc=rc) @@ -556,7 +556,7 @@ subroutine read_static(noahmp, rc) deallocate(flds) else allocate(flds(1)) - write(filename, fmt="(A,I0,A)") trim(noahmp%nmlist%input_dir)//'C', noahmp%domain%ni, '.vegetation_type.tile*.nc' + write(filename, fmt="(A,I0,A)") trim(noahmp%nmlist%fixed_dir)//'C', noahmp%domain%ni, '.vegetation_type.tile*.nc' flds(1)%short_name = 'vegetation_type' flds(1)%ptr1r4 => tmpr4 call read_tiled_file(noahmp, filename, flds, rc=rc) @@ -580,7 +580,7 @@ subroutine read_static(noahmp, rc) deallocate(flds) else allocate(flds(1)) - write(filename, fmt="(A,I0,A)") trim(noahmp%nmlist%input_dir)//'C', noahmp%domain%ni, '.slope_type.tile*.nc' + write(filename, fmt="(A,I0,A)") trim(noahmp%nmlist%fixed_dir)//'C', noahmp%domain%ni, '.slope_type.tile*.nc' flds(1)%short_name = 'slope_type' flds(1)%ptr1r4 => tmpr4 call read_tiled_file(noahmp, filename, flds, rc=rc) @@ -604,7 +604,7 @@ subroutine read_static(noahmp, rc) deallocate(flds) else allocate(flds(1)) - write(filename, fmt="(A,I0,A)") trim(noahmp%nmlist%input_dir)//'C', noahmp%domain%ni, '.substrate_temperature.tile*.nc' + write(filename, fmt="(A,I0,A)") trim(noahmp%nmlist%fixed_dir)//'C', noahmp%domain%ni, '.substrate_temperature.tile*.nc' flds(1)%short_name = 'substrate_temperature' flds(1)%ptr1r4 => tmpr4 call read_tiled_file(noahmp, filename, flds, rc=rc) @@ -628,7 +628,7 @@ subroutine read_static(noahmp, rc) deallocate(flds) else allocate(flds(1)) - write(filename, fmt="(A,I0,A)") trim(noahmp%nmlist%input_dir)//'C', noahmp%domain%ni, '.maximum_snow_albedo.tile*.nc' + write(filename, fmt="(A,I0,A)") trim(noahmp%nmlist%fixed_dir)//'C', noahmp%domain%ni, '.maximum_snow_albedo.tile*.nc' flds(1)%short_name = 'maximum_snow_albedo' flds(1)%ptr1r4 => tmpr4 call read_tiled_file(noahmp, filename, flds, rc=rc) @@ -642,7 +642,7 @@ subroutine read_static(noahmp, rc) !---------------------- allocate(flds(1)) - write(filename, fmt="(A,I0,A)") trim(noahmp%nmlist%input_dir)//'C', noahmp%domain%ni, '.vegetation_greenness.tile*.nc' + write(filename, fmt="(A,I0,A)") trim(noahmp%nmlist%fixed_dir)//'C', noahmp%domain%ni, '.vegetation_greenness.tile*.nc' flds(1)%short_name = 'vegetation_greenness' flds(1)%nrec = 12; flds(1)%ptr2r4 => tmp2r4 call read_tiled_file(noahmp, filename, flds, rc=rc) @@ -657,7 +657,7 @@ subroutine read_static(noahmp, rc) !---------------------- allocate(flds(1)) - write(filename, fmt="(A,I0,A)") trim(noahmp%nmlist%input_dir)//'C', noahmp%domain%ni, '.soil_color.tile*.nc' + write(filename, fmt="(A,I0,A)") trim(noahmp%nmlist%fixed_dir)//'C', noahmp%domain%ni, '.soil_color.tile*.nc' flds(1)%short_name = 'soil_color' flds(1)%ptr1r4 => tmpr4 call read_tiled_file(noahmp, filename, flds, rc=rc) diff --git a/drivers/nuopc/lnd_comp_nuopc.F90 b/drivers/nuopc/lnd_comp_nuopc.F90 index c561e52..e208990 100644 --- a/drivers/nuopc/lnd_comp_nuopc.F90 +++ b/drivers/nuopc/lnd_comp_nuopc.F90 @@ -223,6 +223,16 @@ subroutine InitializeRealize(gcomp, importState, exportState, clock, rc) return end if + call NUOPC_CompAttributeGet(gcomp, name='fixed_dir', value=cvalue, isPresent=isPresent, isSet=isSet, rc=rc) + if (ChkErr(rc,__LINE__,u_FILE_u)) return + + if (isPresent .and. isSet) then + noahmp%nmlist%fixed_dir = trim(cvalue) + else + noahmp%nmlist%fixed_dir = "INPUT/" + end if + call ESMF_LogWrite(trim(subname)//': fixed_dir = '//trim(noahmp%nmlist%fixed_dir), ESMF_LOGMSG_INFO) + call NUOPC_CompAttributeGet(gcomp, name='input_dir', value=cvalue, isPresent=isPresent, isSet=isSet, rc=rc) if (ChkErr(rc,__LINE__,u_FILE_u)) return diff --git a/drivers/nuopc/lnd_comp_types.F90 b/drivers/nuopc/lnd_comp_types.F90 index 888ee8f..59501c4 100644 --- a/drivers/nuopc/lnd_comp_types.F90 +++ b/drivers/nuopc/lnd_comp_types.F90 @@ -245,6 +245,7 @@ module lnd_comp_types character*100 :: case_name ! name of case character*255 :: mosaic_file ! name of mosaic file character*255 :: input_dir ! input directory for tiled files + character*255 :: fixed_dir ! fixed input directory for tiled files character*255 :: restart_dir ! restart directory character*255 :: restart_file ! restart file name character*255 :: ic_type ! source of initial conditions, custom vs. sfc From 432435202a88f3b225af08dc0e701ae2401944b2 Mon Sep 17 00:00:00 2001 From: Ufuk Turuncoglu Date: Thu, 7 Nov 2024 23:26:55 -0600 Subject: [PATCH 5/7] update ccpp related files and coupling interface --- drivers/ccpp/noahmpdrv.F90 | 6 ++++-- drivers/ccpp/noahmpdrv.meta | 7 +++++++ drivers/ccpp/update.sh | 35 ++++++++++++++++++++++--------- drivers/nuopc/lnd_comp_driver.F90 | 2 +- drivers/nuopc/lnd_comp_types.F90 | 3 ++- 5 files changed, 39 insertions(+), 14 deletions(-) diff --git a/drivers/ccpp/noahmpdrv.F90 b/drivers/ccpp/noahmpdrv.F90 index 1313e9f..56b2706 100644 --- a/drivers/ccpp/noahmpdrv.F90 +++ b/drivers/ccpp/noahmpdrv.F90 @@ -127,7 +127,8 @@ end subroutine noahmpdrv_init subroutine noahmpdrv_run & !................................... ! --- inputs: - ( im, km, lsnowl, itime, ps, u1, v1, t1, q1, soiltyp,soilcol,& + ( im, km, lsnowl, itime, flag_init, ps, u1, v1, t1, q1, & + soiltyp,soilcol, & vegtype, sigmaf, dlwflx, dswsfc, snet, delt, tg3, cm, ch, & prsl1, prslk1, prslki, prsik1, zf,pblh, dry, wind, slopetyp,& shdmin, shdmax, snoalb, sfalb, flag_iter,con_g, & @@ -233,6 +234,7 @@ subroutine noahmpdrv_run & integer , intent(in) :: km ! vertical soil layer dimension integer , intent(in) :: lsnowl ! lower bound for snow level arrays integer , intent(in) :: itime ! NOT USED + logical , intent(in) :: flag_init ! flag signaling first time step real(kind=kind_phys), dimension(:) , intent(in) :: ps ! surface pressure [Pa] real(kind=kind_phys), dimension(:) , intent(in) :: u1 ! u-component of wind [m/s] real(kind=kind_phys), dimension(:) , intent(in) :: v1 ! u-component of wind [m/s] @@ -693,7 +695,7 @@ subroutine noahmpdrv_run & ! ! --- Just return if external land component is activated for two-way interaction ! - if (cpllnd .and. cpllnd2atm) return + if (cpllnd .and. cpllnd2atm .and. (.not. flag_init)) return do i = 1, im diff --git a/drivers/ccpp/noahmpdrv.meta b/drivers/ccpp/noahmpdrv.meta index 7535500..75bd4d7 100644 --- a/drivers/ccpp/noahmpdrv.meta +++ b/drivers/ccpp/noahmpdrv.meta @@ -129,6 +129,13 @@ dimensions = () type = integer intent = in +[flag_init] + standard_name = flag_for_first_timestep + long_name = flag signaling first time step for time integration loop + units = flag + dimensions = () + type = logical + intent = in [ps] standard_name = surface_air_pressure long_name = surface pressure diff --git a/drivers/ccpp/update.sh b/drivers/ccpp/update.sh index b5d6ccd..65eeb80 100755 --- a/drivers/ccpp/update.sh +++ b/drivers/ccpp/update.sh @@ -1,24 +1,39 @@ #!/bin/bash -cp -f ../../../../FV3/ccpp/physics/physics/tools/funcphys.f90 . +echo "checksum - before" md5sum ../../../../FV3/ccpp/physics/physics/tools/funcphys.f90 funcphys.f90 -cp -f ../../../../FV3/ccpp/physics/physics/hooks/machine.F . md5sum ../../../../FV3/ccpp/physics/physics/hooks/machine.F machine.F -cp -f ../../../../FV3/ccpp/physics/physics/hooks/physcons.F90 . md5sum ../../../../FV3/ccpp/physics/physics/hooks/physcons.F90 physcons.F90 -cp -f ../../../../FV3/ccpp/physics/physics/SFC_Models/Land/Noah/namelist_soilveg.f . md5sum ../../../../FV3/ccpp/physics/physics/SFC_Models/Land/Noah/namelist_soilveg.f namelist_soilveg.f -cp -f ../../../../FV3/ccpp/physics/physics/SFC_Models/Land/Noah/set_soilveg.f . md5sum ../../../../FV3/ccpp/physics/physics/SFC_Models/Land/Noah/set_soilveg.f set_soilveg.f -cp -f ../../../../FV3/ccpp/physics/physics/SFC_Layer/UFS/sfc_diff.f . md5sum ../../../../FV3/ccpp/physics/physics/SFC_Layer/UFS/sfc_diff.f sfc_diff.f -cp -f ../../../../FV3/ccpp/physics/physics/SFC_Models/Land/Noahmp/noahmp_tables.f90 . md5sum ../../../../FV3/ccpp/physics/physics/SFC_Models/Land/Noahmp/noahmp_tables.f90 noahmp_tables.f90 -cp -f ../../../../FV3/ccpp/physics/physics/SFC_Models/Land/Noahmp/noahmpdrv.F90 . md5sum ../../../../FV3/ccpp/physics/physics/SFC_Models/Land/Noahmp/noahmpdrv.F90 noahmpdrv.F90 -cp -f ../../../../FV3/ccpp/physics/physics/SFC_Models/Land/Noahmp/noahmpdrv.meta . md5sum ../../../../FV3/ccpp/physics/physics/SFC_Models/Land/Noahmp/noahmpdrv.meta noahmpdrv.meta -cp -f ../../../../FV3/ccpp/physics/physics/SFC_Models/Land/Noahmp/module_sf_noahmp_glacier.F90 ../../src/. md5sum ../../../../FV3/ccpp/physics/physics/SFC_Models/Land/Noahmp/module_sf_noahmp_glacier.F90 ../../src/module_sf_noahmp_glacier.F90 +md5sum ../../../../FV3/ccpp/physics/physics/SFC_Models/Land/Noahmp/module_sf_noahmplsm.F90 ../../src/module_sf_noahmplsm.F90 + +cp -f ../../../../FV3/ccpp/physics/physics/tools/funcphys.f90 . +cp -f ../../../../FV3/ccpp/physics/physics/hooks/machine.F . +cp -f ../../../../FV3/ccpp/physics/physics/hooks/physcons.F90 . +cp -f ../../../../FV3/ccpp/physics/physics/SFC_Models/Land/Noah/namelist_soilveg.f . +cp -f ../../../../FV3/ccpp/physics/physics/SFC_Models/Land/Noah/set_soilveg.f . +cp -f ../../../../FV3/ccpp/physics/physics/SFC_Layer/UFS/sfc_diff.f . +cp -f ../../../../FV3/ccpp/physics/physics/SFC_Models/Land/Noahmp/noahmp_tables.f90 . +cp -f ../../../../FV3/ccpp/physics/physics/SFC_Models/Land/Noahmp/noahmpdrv.F90 . +cp -f ../../../../FV3/ccpp/physics/physics/SFC_Models/Land/Noahmp/noahmpdrv.meta . +cp -f ../../../../FV3/ccpp/physics/physics/SFC_Models/Land/Noahmp/module_sf_noahmp_glacier.F90 ../../src/. cp -f ../../../../FV3/ccpp/physics/physics/SFC_Models/Land/Noahmp/module_sf_noahmplsm.F90 ../../src/. + +echo "checksum - after" +md5sum ../../../../FV3/ccpp/physics/physics/tools/funcphys.f90 funcphys.f90 +md5sum ../../../../FV3/ccpp/physics/physics/hooks/machine.F machine.F +md5sum ../../../../FV3/ccpp/physics/physics/hooks/physcons.F90 physcons.F90 +md5sum ../../../../FV3/ccpp/physics/physics/SFC_Models/Land/Noah/namelist_soilveg.f namelist_soilveg.f +md5sum ../../../../FV3/ccpp/physics/physics/SFC_Models/Land/Noah/set_soilveg.f set_soilveg.f +md5sum ../../../../FV3/ccpp/physics/physics/SFC_Layer/UFS/sfc_diff.f sfc_diff.f +md5sum ../../../../FV3/ccpp/physics/physics/SFC_Models/Land/Noahmp/noahmp_tables.f90 noahmp_tables.f90 +md5sum ../../../../FV3/ccpp/physics/physics/SFC_Models/Land/Noahmp/noahmpdrv.F90 noahmpdrv.F90 +md5sum ../../../../FV3/ccpp/physics/physics/SFC_Models/Land/Noahmp/noahmpdrv.meta noahmpdrv.meta +md5sum ../../../../FV3/ccpp/physics/physics/SFC_Models/Land/Noahmp/module_sf_noahmp_glacier.F90 ../../src/module_sf_noahmp_glacier.F90 md5sum ../../../../FV3/ccpp/physics/physics/SFC_Models/Land/Noahmp/module_sf_noahmplsm.F90 ../../src/module_sf_noahmplsm.F90 diff --git a/drivers/nuopc/lnd_comp_driver.F90 b/drivers/nuopc/lnd_comp_driver.F90 index a788815..ecd3f1b 100644 --- a/drivers/nuopc/lnd_comp_driver.F90 +++ b/drivers/nuopc/lnd_comp_driver.F90 @@ -600,7 +600,7 @@ subroutine drv_run(gcomp, noahmp, rc) call noahmpdrv_run( & ! --- inputs: noahmp%static%im , noahmp%static%km , noahmp%static%lsnowl , & - noahmp%static%itime , & + noahmp%static%itime , noahmp%static%flag_init, & noahmp%forc%ps , noahmp%model%u1 , noahmp%model%v1 , & noahmp%forc%t1 , noahmp%forc%q1 , noahmp%model%soiltyp , & noahmp%model%soilcol , & diff --git a/drivers/nuopc/lnd_comp_types.F90 b/drivers/nuopc/lnd_comp_types.F90 index d52be01..4dd7206 100644 --- a/drivers/nuopc/lnd_comp_types.F90 +++ b/drivers/nuopc/lnd_comp_types.F90 @@ -55,6 +55,7 @@ module lnd_comp_types integer :: km ! vertical soil layer dimension integer :: lsnowl = -2 ! lower bound of vertical dimension of surface snow integer :: itime = huge(1)! not used, set to something huge by default + logical :: flag_init = .false. ! not used in component model integer :: isot ! sfc soil type data source zobler or statsgo integer :: ivegsrc ! sfc veg type data source umd or igbp integer :: idveg ! option for dynamic vegetation @@ -281,7 +282,7 @@ module lnd_comp_types real(kind=r8) :: initial_emiss ! initial value for the emissivity (constant in everywhere) real(kind=r8) :: initial_albedo ! initial value for the monthly albedo (constant in everywhere) character*10 :: decomp_type ! decomposition type: default or custom - logical :: debug_level ! debug level + integer :: debug_level ! debug level end type namelist_type type noahmp_type From bc97a71966d701c1d0f7abeb612df3b84119dced Mon Sep 17 00:00:00 2001 From: Ufuk Turuncoglu Date: Fri, 27 Dec 2024 17:18:39 -0600 Subject: [PATCH 6/7] revert flag_init changes --- drivers/ccpp/noahmpdrv.F90 | 5 ++--- drivers/ccpp/noahmpdrv.meta | 7 ------- drivers/nuopc/lnd_comp_driver.F90 | 2 +- drivers/nuopc/lnd_comp_types.F90 | 1 - 4 files changed, 3 insertions(+), 12 deletions(-) diff --git a/drivers/ccpp/noahmpdrv.F90 b/drivers/ccpp/noahmpdrv.F90 index 46749e5..431ff35 100644 --- a/drivers/ccpp/noahmpdrv.F90 +++ b/drivers/ccpp/noahmpdrv.F90 @@ -420,7 +420,7 @@ end subroutine noahmpdrv_finalize subroutine noahmpdrv_run & !................................... ! --- inputs: - ( im, km, lsnowl, itime, flag_init, ps, u1, v1, t1, q1, & + ( im, km, lsnowl, itime, ps, u1, v1, t1, q1, & soiltyp,soilcol, & vegtype, sigmaf, dlwflx, dswsfc, snet, delt, tg3, cm, ch, & prsl1, prslk1, prslki, prsik1, zf,pblh, dry, wind, slopetyp,& @@ -527,7 +527,6 @@ subroutine noahmpdrv_run & integer , intent(in) :: km ! vertical soil layer dimension integer , intent(in) :: lsnowl ! lower bound for snow level arrays integer , intent(in) :: itime ! NOT USED - logical , intent(in) :: flag_init ! flag signaling first time step real(kind=kind_phys), dimension(:) , intent(in) :: ps ! surface pressure [Pa] real(kind=kind_phys), dimension(:) , intent(in) :: u1 ! u-component of wind [m/s] real(kind=kind_phys), dimension(:) , intent(in) :: v1 ! u-component of wind [m/s] @@ -988,7 +987,7 @@ subroutine noahmpdrv_run & ! ! --- Just return if external land component is activated for two-way interaction ! - if (cpllnd .and. cpllnd2atm .and. (.not. flag_init)) return + if (cpllnd .and. cpllnd2atm) return do i = 1, im diff --git a/drivers/ccpp/noahmpdrv.meta b/drivers/ccpp/noahmpdrv.meta index 1c08d9a..7d1150c 100644 --- a/drivers/ccpp/noahmpdrv.meta +++ b/drivers/ccpp/noahmpdrv.meta @@ -356,13 +356,6 @@ dimensions = () type = integer intent = in -[flag_init] - standard_name = flag_for_first_timestep - long_name = flag signaling first time step for time integration loop - units = flag - dimensions = () - type = logical - intent = in [ps] standard_name = surface_air_pressure long_name = surface pressure diff --git a/drivers/nuopc/lnd_comp_driver.F90 b/drivers/nuopc/lnd_comp_driver.F90 index ecd3f1b..a788815 100644 --- a/drivers/nuopc/lnd_comp_driver.F90 +++ b/drivers/nuopc/lnd_comp_driver.F90 @@ -600,7 +600,7 @@ subroutine drv_run(gcomp, noahmp, rc) call noahmpdrv_run( & ! --- inputs: noahmp%static%im , noahmp%static%km , noahmp%static%lsnowl , & - noahmp%static%itime , noahmp%static%flag_init, & + noahmp%static%itime , & noahmp%forc%ps , noahmp%model%u1 , noahmp%model%v1 , & noahmp%forc%t1 , noahmp%forc%q1 , noahmp%model%soiltyp , & noahmp%model%soilcol , & diff --git a/drivers/nuopc/lnd_comp_types.F90 b/drivers/nuopc/lnd_comp_types.F90 index 4dd7206..221939b 100644 --- a/drivers/nuopc/lnd_comp_types.F90 +++ b/drivers/nuopc/lnd_comp_types.F90 @@ -55,7 +55,6 @@ module lnd_comp_types integer :: km ! vertical soil layer dimension integer :: lsnowl = -2 ! lower bound of vertical dimension of surface snow integer :: itime = huge(1)! not used, set to something huge by default - logical :: flag_init = .false. ! not used in component model integer :: isot ! sfc soil type data source zobler or statsgo integer :: ivegsrc ! sfc veg type data source umd or igbp integer :: idveg ! option for dynamic vegetation From 559ce9c1cf00e4569ddeb9461b9649e33b2efb43 Mon Sep 17 00:00:00 2001 From: Ufuk Turuncoglu Date: Tue, 7 Jan 2025 13:38:01 -0600 Subject: [PATCH 7/7] sync with ccpp version --- drivers/ccpp/noahmpdrv.F90 | 3 +-- 1 file changed, 1 insertion(+), 2 deletions(-) diff --git a/drivers/ccpp/noahmpdrv.F90 b/drivers/ccpp/noahmpdrv.F90 index 431ff35..d4971ef 100644 --- a/drivers/ccpp/noahmpdrv.F90 +++ b/drivers/ccpp/noahmpdrv.F90 @@ -420,8 +420,7 @@ end subroutine noahmpdrv_finalize subroutine noahmpdrv_run & !................................... ! --- inputs: - ( im, km, lsnowl, itime, ps, u1, v1, t1, q1, & - soiltyp,soilcol, & + ( im, km, lsnowl, itime, ps, u1, v1, t1, q1, soiltyp,soilcol,& vegtype, sigmaf, dlwflx, dswsfc, snet, delt, tg3, cm, ch, & prsl1, prslk1, prslki, prsik1, zf,pblh, dry, wind, slopetyp,& shdmin, shdmax, snoalb, sfalb, flag_iter,con_g, &