Skip to content
New issue

Have a question about this project? Sign up for a free GitHub account to open an issue and contact its maintainers and the community.

Sign up for GitHub

By clicking “Sign up for GitHub”, you agree to our terms of service and privacy statement. We’ll occasionally send you account related emails.

Already on GitHub? Sign in to your account

Enable new decomposition algorithm #13

Open
wants to merge 9 commits into
base: develop
Choose a base branch
from
+331 −30
Open

Enable new decomposition algorithm #13

Show file tree
Hide file tree
Changes from all commits
Commits
Show all changes
9 commits
Select commit Hold shift + click to select a range
a19b899
distribute land and ocean points evenly across processors
uturuncoglu May 23, 2024
35e6bd0
clean implementation
uturuncoglu May 24, 2024
97e5dc7
minor fix for debug
uturuncoglu Jul 22, 2024
d765daf
add option to specify fixed file directory
uturuncoglu Aug 2, 2024
c700699
Merge remote-tracking branch 'origin/develop' into feature/perf
uturuncoglu Sep 6, 2024
4324352
update ccpp related files and coupling interface
uturuncoglu Nov 8, 2024
adaf118
Merge remote-tracking branch 'origin/develop' into feature/perf
uturuncoglu Dec 6, 2024
bc97a71
revert flag_init changes
uturuncoglu Dec 27, 2024
559ce9c
sync with ccpp version
uturuncoglu Jan 7, 2025
File filter

Filter by extension

Filter by extension
Conversations
Failed to load comments.
Loading
Jump to
Jump to file
Failed to load files.
Loading
Diff view
Diff view
35 changes: 25 additions & 10 deletions drivers/ccpp/update.sh
View file
Open in desktop
Original file line number Diff line number Diff line change
@@ -1,24 +1,39 @@
#!/bin/bash

cp -f ../../../../FV3/ccpp/physics/physics/tools/funcphys.f90 .
echo "checksum - before"
md5sum ../../../../FV3/ccpp/physics/physics/tools/funcphys.f90 funcphys.f90
cp -f ../../../../FV3/ccpp/physics/physics/hooks/machine.F .
md5sum ../../../../FV3/ccpp/physics/physics/hooks/machine.F machine.F
cp -f ../../../../FV3/ccpp/physics/physics/hooks/physcons.F90 .
md5sum ../../../../FV3/ccpp/physics/physics/hooks/physcons.F90 physcons.F90
cp -f ../../../../FV3/ccpp/physics/physics/SFC_Models/Land/Noah/namelist_soilveg.f .
md5sum ../../../../FV3/ccpp/physics/physics/SFC_Models/Land/Noah/namelist_soilveg.f namelist_soilveg.f
cp -f ../../../../FV3/ccpp/physics/physics/SFC_Models/Land/Noah/set_soilveg.f .
md5sum ../../../../FV3/ccpp/physics/physics/SFC_Models/Land/Noah/set_soilveg.f set_soilveg.f
cp -f ../../../../FV3/ccpp/physics/physics/SFC_Layer/UFS/sfc_diff.f .
md5sum ../../../../FV3/ccpp/physics/physics/SFC_Layer/UFS/sfc_diff.f sfc_diff.f
cp -f ../../../../FV3/ccpp/physics/physics/SFC_Models/Land/Noahmp/noahmp_tables.f90 .
md5sum ../../../../FV3/ccpp/physics/physics/SFC_Models/Land/Noahmp/noahmp_tables.f90 noahmp_tables.f90
cp -f ../../../../FV3/ccpp/physics/physics/SFC_Models/Land/Noahmp/noahmpdrv.F90 .
md5sum ../../../../FV3/ccpp/physics/physics/SFC_Models/Land/Noahmp/noahmpdrv.F90 noahmpdrv.F90
cp -f ../../../../FV3/ccpp/physics/physics/SFC_Models/Land/Noahmp/noahmpdrv.meta .
md5sum ../../../../FV3/ccpp/physics/physics/SFC_Models/Land/Noahmp/noahmpdrv.meta noahmpdrv.meta
cp -f ../../../../FV3/ccpp/physics/physics/SFC_Models/Land/Noahmp/module_sf_noahmp_glacier.F90 ../../src/.
md5sum ../../../../FV3/ccpp/physics/physics/SFC_Models/Land/Noahmp/module_sf_noahmp_glacier.F90 ../../src/module_sf_noahmp_glacier.F90
md5sum ../../../../FV3/ccpp/physics/physics/SFC_Models/Land/Noahmp/module_sf_noahmplsm.F90 ../../src/module_sf_noahmplsm.F90

cp -f ../../../../FV3/ccpp/physics/physics/tools/funcphys.f90 .
cp -f ../../../../FV3/ccpp/physics/physics/hooks/machine.F .
cp -f ../../../../FV3/ccpp/physics/physics/hooks/physcons.F90 .
cp -f ../../../../FV3/ccpp/physics/physics/SFC_Models/Land/Noah/namelist_soilveg.f .
cp -f ../../../../FV3/ccpp/physics/physics/SFC_Models/Land/Noah/set_soilveg.f .
cp -f ../../../../FV3/ccpp/physics/physics/SFC_Layer/UFS/sfc_diff.f .
cp -f ../../../../FV3/ccpp/physics/physics/SFC_Models/Land/Noahmp/noahmp_tables.f90 .
cp -f ../../../../FV3/ccpp/physics/physics/SFC_Models/Land/Noahmp/noahmpdrv.F90 .
cp -f ../../../../FV3/ccpp/physics/physics/SFC_Models/Land/Noahmp/noahmpdrv.meta .
cp -f ../../../../FV3/ccpp/physics/physics/SFC_Models/Land/Noahmp/module_sf_noahmp_glacier.F90 ../../src/.
cp -f ../../../../FV3/ccpp/physics/physics/SFC_Models/Land/Noahmp/module_sf_noahmplsm.F90 ../../src/.

echo "checksum - after"
md5sum ../../../../FV3/ccpp/physics/physics/tools/funcphys.f90 funcphys.f90
md5sum ../../../../FV3/ccpp/physics/physics/hooks/machine.F machine.F
md5sum ../../../../FV3/ccpp/physics/physics/hooks/physcons.F90 physcons.F90
md5sum ../../../../FV3/ccpp/physics/physics/SFC_Models/Land/Noah/namelist_soilveg.f namelist_soilveg.f
md5sum ../../../../FV3/ccpp/physics/physics/SFC_Models/Land/Noah/set_soilveg.f set_soilveg.f
md5sum ../../../../FV3/ccpp/physics/physics/SFC_Layer/UFS/sfc_diff.f sfc_diff.f
md5sum ../../../../FV3/ccpp/physics/physics/SFC_Models/Land/Noahmp/noahmp_tables.f90 noahmp_tables.f90
md5sum ../../../../FV3/ccpp/physics/physics/SFC_Models/Land/Noahmp/noahmpdrv.F90 noahmpdrv.F90
md5sum ../../../../FV3/ccpp/physics/physics/SFC_Models/Land/Noahmp/noahmpdrv.meta noahmpdrv.meta
md5sum ../../../../FV3/ccpp/physics/physics/SFC_Models/Land/Noahmp/module_sf_noahmp_glacier.F90 ../../src/module_sf_noahmp_glacier.F90
md5sum ../../../../FV3/ccpp/physics/physics/SFC_Models/Land/Noahmp/module_sf_noahmplsm.F90 ../../src/module_sf_noahmplsm.F90
252 changes: 249 additions & 3 deletions drivers/nuopc/lnd_comp_domain.F90
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -23,6 +23,7 @@ module lnd_comp_domain
use ESMF , only : ESMF_RouteHandleDestroy, ESMF_GridGet, ESMF_GridGetCoord
use ESMF , only : ESMF_FieldRegridGetArea, ESMF_CoordSys_Flag
use ESMF , only : ESMF_MeshGetFieldBounds, ESMF_COORDSYS_CART, ESMF_KIND_R8
use ESMF , only : ESMF_MeshDestroy, ESMF_DistGridCreate, ESMF_VMAllGatherV
use NUOPC, only : NUOPC_CompAttributeGet

use lnd_comp_kind , only : r4 => shr_kind_r4
Expand Down Expand Up @@ -62,10 +63,11 @@ subroutine lnd_set_decomp_and_domain_from_mosaic(gcomp, noahmp, rc)
! local variables
real(r4), target, allocatable :: tmpr4(:)
real(r8), target, allocatable :: tmpr8(:)
integer :: n
integer :: n, petCount, localPet
integer :: decomptile(2,6)
integer :: maxIndex(2)
type(ESMF_Decomp_Flag) :: decompflagPTile(2,6)
type(ESMF_VM) :: vm

type(field_type) :: flds(1)
integer :: numOwnedElements, spatialDim, rank
Expand All @@ -87,6 +89,16 @@ subroutine lnd_set_decomp_and_domain_from_mosaic(gcomp, noahmp, rc)
rc = ESMF_SUCCESS
call ESMF_LogWrite(subname//' called', ESMF_LOGMSG_INFO)

! ---------------------
! Query VM
! ---------------------

call ESMF_GridCompGet(gcomp, vm=vm, rc=rc)
if (ChkErr(rc,__LINE__,u_FILE_u)) return

call ESMF_VMGet(vm, petCount=petCount, localPet=localPet, rc=rc)
if (ChkErr(rc,__LINE__,u_FILE_u)) return

! ---------------------
! Set decomposition and decide it is regional or global
! ---------------------
Expand All @@ -101,7 +113,14 @@ subroutine lnd_set_decomp_and_domain_from_mosaic(gcomp, noahmp, rc)
! set number of tiles
noahmp%domain%ntiles = 6

! set decomposition
! check user provided layout
if (petCount /= noahmp%domain%layout(1)*noahmp%domain%layout(2)*noahmp%domain%ntiles) then
call ESMF_LogWrite(trim(subname)//": ERROR in layout. layout_x * layout_y * 6 != #PETs", ESMF_LOGMSG_INFO)
rc = ESMF_FAILURE
return
end if

! use user provided layout to set decomposition
do n = 1, noahmp%domain%ntiles
decomptile(1,n) = noahmp%domain%layout(1)
decomptile(2,n) = noahmp%domain%layout(2)
Expand Down Expand Up @@ -226,7 +245,7 @@ subroutine lnd_set_decomp_and_domain_from_mosaic(gcomp, noahmp, rc)
if (ChkErr(rc,__LINE__,u_FILE_u)) return
vegtype(:) = int(tmpr8)
else
write(filename, fmt="(A,I0,A)") trim(noahmp%nmlist%input_dir)//'C',noahmp%domain%ni, '.vegetation_type.tile*.nc'
write(filename, fmt="(A,I0,A)") trim(noahmp%nmlist%fixed_dir)//'C',noahmp%domain%ni, '.vegetation_type.tile*.nc'
flds(1)%short_name = 'vegetation_type'
flds(1)%ptr1r4 => tmpr4
call read_tiled_file(noahmp, filename, flds, rc=rc)
Expand Down Expand Up @@ -257,6 +276,16 @@ subroutine lnd_set_decomp_and_domain_from_mosaic(gcomp, noahmp, rc)
call ESMF_MeshSet(noahmp%domain%mesh, elementMask=noahmp%domain%mask, rc=rc)
if (chkerr(rc,__LINE__,u_FILE_u)) return

! ---------------------
! Modify decomposition to evenly distribute land and ocean points
! ---------------------

if (trim(noahmp%nmlist%decomp_type) == 'custom') then
! modify decomposition
call lnd_modify_decomp(gcomp, noahmp, rc)
if (chkerr(rc,__LINE__,u_FILE_u)) return
end if

! ---------------------
! Get height from orography file
! ---------------------
Expand Down Expand Up @@ -339,4 +368,221 @@ subroutine lnd_set_decomp_and_domain_from_mosaic(gcomp, noahmp, rc)

end subroutine lnd_set_decomp_and_domain_from_mosaic

!===============================================================================
subroutine lnd_modify_decomp(gcomp, noahmp, rc)

! input/output variables
type(ESMF_GridComp), intent(in) :: gcomp
type(noahmp_type), intent(inout) :: noahmp
integer, intent(out) :: rc

! local variables
type(ESMF_VM) :: vm
type(ESMF_Mesh) :: mesh
type(ESMF_DistGrid) :: distgrid, distgrid_new
type(field_type) :: flds(1)
integer :: n, m, g
integer :: petCount, localPet
integer :: lsize, gsize(1)
integer :: nlnd, nocn
integer :: begl_l, endl_l, begl_o, endl_o
integer, allocatable :: nlnd_loc(:)
integer, allocatable :: nocn_loc(:)
integer, allocatable :: mask_glb(:)
integer, allocatable :: gindex_loc(:)
integer, allocatable :: gindex_glb(:)
integer, allocatable :: gindex_lnd(:)
integer, allocatable :: gindex_ocn(:)
integer, allocatable :: gindex_new(:)
integer, allocatable :: lsize_arr(:)
real(r4), target, allocatable :: tmpr4(:)
character(len=cl) :: msg, filename
character(len=*), parameter :: subname = trim(modName)//':(lnd_modify_decomp) '
!-------------------------------------------------------------------------------

rc = ESMF_SUCCESS
call ESMF_LogWrite(subname//' called', ESMF_LOGMSG_INFO)

! ---------------------
! Query VM
! ---------------------

call ESMF_GridCompGet(gcomp, vm=vm, rc=rc)
if (ChkErr(rc,__LINE__,u_FILE_u)) return

call ESMF_VMGet(vm, petCount=petCount, localPet=localPet, rc=rc)
if (ChkErr(rc,__LINE__,u_FILE_u)) return

! ---------------------
! Query existing mesh
! ---------------------

! retrive default distgrid
call ESMF_MeshGet(noahmp%domain%mesh, elementdistGrid=distgrid, rc=rc)
if (chkerr(rc,__LINE__,u_FILE_u)) return

! get local number of elements
call ESMF_DistGridGet(distgrid, localDe=0, elementCount=lsize, rc=rc)
if (chkerr(rc,__LINE__,u_FILE_u)) return

! calculate number of elements globally
call ESMF_VMAllReduce(vm, (/ lsize /), gsize, 1, ESMF_REDUCE_SUM, rc=rc)
if (chkerr(rc,__LINE__,u_FILE_u)) return

nlnd = count(noahmp%domain%mask(:) > 0, dim=1)
nocn = lsize-nlnd
write(msg, fmt='(A,4I8)') trim(subname)//' : lsize, gsize, nlnd, nocn = ', &
lsize, gsize(1), nlnd, nocn
call ESMF_LogWrite(trim(msg), ESMF_LOGMSG_INFO)

! get default sequence index
allocate(gindex_loc(lsize))
call ESMF_DistGridGet(distgrid, 0, seqIndexList=gindex_loc, rc=rc)
if (chkerr(rc,__LINE__,u_FILE_u)) return

! ---------------------
! Create global view of indexes and mask
! --------------------

! collect local sizes
allocate(lsize_arr(petCount))
call ESMF_VMAllGatherV(vm, sendData=(/ lsize /), sendCount=1, &
recvData=lsize_arr, recvCounts=(/ (1, n = 0, petCount-1) /), &
recvOffsets=(/ (n, n = 0, petCount-1) /), rc=rc)
if (chkerr(rc,__LINE__,u_FILE_u)) return

! global view of indexes
allocate(gindex_glb(gsize(1)))
gindex_glb(:) = 0
call ESMF_VMAllGatherV(vm, sendData=gindex_loc, sendCount=lsize, &
recvData=gindex_glb, recvCounts=lsize_arr, &
recvOffsets=(/ (sum(lsize_arr(1:n))-lsize_arr(1), n = 1, petCount) /), rc=rc)
if (chkerr(rc,__LINE__,u_FILE_u)) return

! global view of land sea mask
allocate(mask_glb(gsize(1)))
mask_glb(:) = 0
call ESMF_VMAllGatherV(vm, sendData=noahmp%domain%mask, sendCount=lsize, &
recvData=mask_glb, recvCounts=lsize_arr, &
recvOffsets=(/ (sum(lsize_arr(1:n))-lsize_arr(1), n = 1, petCount) /), rc=rc)
if (chkerr(rc,__LINE__,u_FILE_u)) return

! ---------------------
! split global indexes as land and ocean
! ---------------------

nlnd = count(mask_glb > 0, dim=1)
nocn = gsize(1)-nlnd
allocate(gindex_lnd(nlnd))
gindex_lnd = 0
allocate(gindex_ocn(nocn))
gindex_ocn = 0

n = 0
m = 0
do g = 1, gsize(1)
if (mask_glb(g) > 0) then
n = n+1
gindex_lnd(n) = gindex_glb(g)
else
m = m+1
gindex_ocn(m) = gindex_glb(g)
end if
end do

! ---------------------
! create new local indexes
! ---------------------

allocate(nlnd_loc(0:petCount-1))
nlnd_loc = 0
allocate(nocn_loc(0:petCount-1))
nocn_loc = 0
do n = 0, petCount-1
nlnd_loc(n) = nlnd/petCount
nocn_loc(n) = nocn/petCount
if (n < mod(nlnd, petCount)) then
nlnd_loc(n) = nlnd_loc(n)+1
else
nocn_loc(n) = nocn_loc(n)+1
end if
end do
if (localPet == 0) then
begl_l = 1
begl_o = 1
else
begl_l = sum(nlnd_loc(0:localPet-1))+1
begl_o = sum(nocn_loc(0:localPet-1))+1
end if
endl_l = sum(nlnd_loc(0:localPet))
endl_o = sum(nocn_loc(0:localPet))

write(msg,'(A,6I8)') trim(subname)//' : nlnd_loc, nocn_loc, begl_l, endl_l, begl_o, endl_o = ', &
nlnd_loc(localPet), nocn_loc(localPet), begl_l, endl_l, begl_o, endl_o
call ESMF_LogWrite(trim(msg), ESMF_LOGMSG_INFO)

allocate(gindex_new(nlnd_loc(localPet)+nocn_loc(localPet)))
gindex_new(:nlnd_loc(localPet)) = gindex_lnd(begl_l:begl_l)
gindex_new(nlnd_loc(localPet)+1:) = gindex_ocn(begl_o:endl_o)

if (noahmp%nmlist%debug_level > 10) then
do n = 1, nlnd_loc(localPet)+nocn_loc(localPet)
write(msg,'(A,2I8)') trim(subname)//' : n, gindex_new = ', n, gindex_new(n)
call ESMF_LogWrite(trim(msg), ESMF_LOGMSG_INFO)
end do
end if

! ---------------------
! Update mesh with new decomposition
! ---------------------

! create new distgrid with new index
distgrid_new = ESMF_DistGridCreate(arbSeqIndexList=gindex_new, rc=rc)
if (chkerr(rc,__LINE__,u_FILE_u)) return

! create new mesh with new distgrid
mesh = ESMF_MeshCreate(noahmp%domain%mesh, elementDistGrid=distgrid_new, rc=rc)
if (chkerr(rc,__LINE__,u_FILE_u)) return

! destroy old and replace with new one
call ESMF_MeshDestroy(noahmp%domain%mesh, rc=rc)
if (chkerr(rc,__LINE__,u_FILE_u)) return
noahmp%domain%mesh = mesh

! ---------------------
! fix mask and fraction to be consistent with new decomposition
! ---------------------

! mask
noahmp%domain%mask(:nlnd_loc(localPet)) = 1
noahmp%domain%mask(nlnd_loc(localPet)+1:) = 0

! fraction, read from file again
allocate(tmpr4(noahmp%domain%begl:noahmp%domain%endl))
tmpr4(:) = 0.0
filename = trim(noahmp%nmlist%input_dir)//'oro_data.tile*.nc'
flds(1)%short_name = 'land_frac'
flds(1)%ptr1r4 => tmpr4
call read_tiled_file(noahmp, filename, flds, rc=rc)
if (ChkErr(rc,__LINE__,u_FILE_u)) return
deallocate(tmpr4)

! ---------------------
! Clean memory
! ---------------------

deallocate(nlnd_loc)
deallocate(nocn_loc)
deallocate(mask_glb)
deallocate(gindex_loc)
deallocate(gindex_glb)
deallocate(gindex_lnd)
deallocate(gindex_ocn)
deallocate(gindex_new)
deallocate(lsize_arr)

call ESMF_LogWrite(subname//' done', ESMF_LOGMSG_INFO)

end subroutine lnd_modify_decomp

end module lnd_comp_domain
Loading
Loading