From 49bbf7900c87270926197768bd050c7b461a8e55 Mon Sep 17 00:00:00 2001 From: Amidu Oloso Date: Tue, 14 Jan 2025 16:54:53 -0500 Subject: [PATCH 1/4] still trying to fix GOCART2G array indexing --- generic/OpenMP_Support.F90 | 22 ++++++++++++++++++++-- 1 file changed, 20 insertions(+), 2 deletions(-) diff --git a/generic/OpenMP_Support.F90 b/generic/OpenMP_Support.F90 index 9cdf0a3ac1c8..661b5b0ace9a 100644 --- a/generic/OpenMP_Support.F90 +++ b/generic/OpenMP_Support.F90 @@ -6,6 +6,7 @@ module MAPL_OpenMP_Support use MAPL_maplgrid use MAPL_ExceptionHandling use mapl_KeywordEnforcerMod + use MAPL_BaseMod, only : MAPL_Grid_Interior !$ use omp_lib implicit none @@ -83,7 +84,7 @@ function make_subgrids_from_bounds(primary_grid, bounds, unusable, rc) result(su type(Interval), intent(in) :: bounds(:) class(KeywordEnforcer), optional, intent(in) :: unusable integer, optional, intent(out) :: rc - integer :: local_count(3) + integer :: local_count(3), global_count(3) integer :: status integer :: petMap(1,1,1) integer :: myPet, section, i, j, k, count, size_ @@ -102,7 +103,7 @@ function make_subgrids_from_bounds(primary_grid, bounds, unusable, rc) result(su !end do allocate(subgrids(size(bounds))) - call MAPL_GridGet(primary_grid,localcellcountPerDim=local_count, _RC) + call MAPL_GridGet(primary_grid,localcellcountPerDim=local_count, globalCellCountPerDim=global_count, _RC) call ESMF_VMGetCurrent(vm, _RC) call ESMF_VMGet(vm, localPET=myPET, _RC) @@ -175,6 +176,23 @@ function make_subgrids_from_bounds(primary_grid, bounds, unusable, rc) result(su itemCount = count, valueList=lons1d, _RC) call ESMF_AttributeSet(subgrids(i), name='GridCornerLats:', & itemCount = count, valueList=lats1d, _RC) + block + integer :: global_grid_info(10) + integer :: i1,i2,j1,j2 + call MAPL_Grid_Interior(primary_grid,i1,i2,j1,j2) + global_grid_info(1:3) = global_count + !global_grid_info(4:6) = local_count + global_grid_info(4) = size(new_lons,1) + global_grid_info(5) = size(new_lons,2) + global_grid_info(6) = local_count(3) + global_grid_info(7) = i1 + global_grid_info(8) = i2 + global_grid_info(9) = j1 + bounds(i)%min - 1 + global_grid_info(10) = j1 + bounds(i)%max - 1 + print '(a,i6,6i4)', __FILE__, __LINE__, myPet, i, j1, j2, global_grid_info(9), global_grid_info(10) + call ESMF_AttributeSet(subgrids(i), name="GLOBAL_GRID_INFO", & + itemCount=10, valueList=global_grid_info, _RC) + end block deallocate(lons1d, lats1d) deallocate(new_corner_lons, new_corner_lats) From ef5e0245d69388c971174b34faab7a66db820958 Mon Sep 17 00:00:00 2001 From: Benjamin Auer Date: Wed, 15 Jan 2025 11:31:47 -0500 Subject: [PATCH 2/4] more updates --- base/Base/Base_Base_implementation.F90 | 60 ++++++++++++++++++++------ base/MaplGrid.F90 | 9 ++++ 2 files changed, 57 insertions(+), 12 deletions(-) diff --git a/base/Base/Base_Base_implementation.F90 b/base/Base/Base_Base_implementation.F90 index b50e84e98897..6653bf2e38fa 100644 --- a/base/Base/Base_Base_implementation.F90 +++ b/base/Base/Base_Base_implementation.F90 @@ -1557,11 +1557,24 @@ module subroutine MAPL_GRID_INTERIOR(GRID,I1,IN,J1,JN) integer :: gridRank integer, allocatable :: localDeToDeMap(:) integer :: rc + logical :: isPresent + integer :: global_grid_info(10) i1=-1 j1=-1 in=-1 jn=-1 + + call ESMF_AttributeGet(grid, name="GLOBAL_GRID_INFO", isPresent=isPresent, _RC) + if (isPresent) then + call ESMF_AttributeGet(grid, name="GLOBAL_GRID_INFO", valueList=global_grid_info, _RC) + I1 = global_grid_info(7) + IN = global_grid_info(8) + j1 = global_grid_info(9) + JN = global_grid_info(10) + _RETURN(_SUCCESS) + end if + call ESMF_GridGet (GRID, dimCount=gridRank, distGrid=distGrid, _RC) call ESMF_DistGridGet(distGRID, delayout=layout, _RC) call ESMF_DELayoutGet(layout, deCount = nDEs, localDeCount=localDeCount,_RC) @@ -2138,6 +2151,24 @@ module subroutine MAPL_GridGetInterior(GRID,I1,IN,J1,JN) integer :: deId integer :: gridRank integer :: rc + logical :: isPresent + integer :: global_grid_info(10) + + i1=-1 + j1=-1 + in=-1 + jn=-1 + + call ESMF_AttributeGet(grid, name="GLOBAL_GRID_INFO", isPresent=isPresent, _RC) + if (isPresent) then + call ESMF_AttributeGet(grid, name="GLOBAL_GRID_INFO", valueList=global_grid_info, _RC) + I1 = global_grid_info(7) + IN = global_grid_info(8) + j1 = global_grid_info(9) + JN = global_grid_info(10) + _RETURN(_SUCCESS) + end if + call ESMF_GridGet (GRID, dimCount=gridRank, distGrid=distGrid, _RC) call ESMF_DistGridGet(distGRID, delayout=layout, _RC) @@ -2627,9 +2658,9 @@ module subroutine MAPL_GetHorzIJIndex(npts,II,JJ,lon,lat,lonR8,latR8,Grid, rc) tmp_lats = latR8 end if -!AOO change tusing GridType atribute if (im_world*6==jm_world) then - call ESMF_AttributeGet(grid, name='GridType', value=grid_type, _RC) - if(trim(grid_type) == "Cubed-Sphere") then + if (im_world*6==jm_world) then +! call ESMF_AttributeGet(grid, name='GridType', value=grid_type, _RC) +! if(trim(grid_type) == "Cubed-Sphere") then call MAPL_GetGlobalHorzIJIndex(npts, II, JJ, lon=lon, lat=lat, lonR8=lonR8, latR8=latR8, Grid=Grid, _RC) @@ -2868,28 +2899,33 @@ function grid_is_ok(grid) result(OK) type(ESMF_Grid), intent(inout) :: grid logical :: OK integer :: I1, I2, J1, J2, j - real(ESMF_KIND_R8), pointer :: corner_lons(:,:), corner_lats(:,:) + real(ESMF_KIND_R8), allocatable :: corner_lons(:,:), corner_lats(:,:) real(ESMF_KIND_R8), allocatable :: lonRe(:), latRe(:) real(ESMF_KIND_R8), allocatable :: accurate_lat(:), accurate_lon(:) real(ESMF_KIND_R8) :: stretch_factor, target_lon, target_lat, shift0 real :: tolerance + integer :: local_dims(3) tolerance = epsilon(1.0) call MAPL_GridGetInterior(grid,I1,I2,J1,J2) + call MAPL_GridGet(grid, localCellCountPerDim=local_dims, _RC) OK = .true. ! check the edge of face 1 along longitude - call ESMF_GridGetCoord(grid,localDE=0,coordDim=1,staggerloc=ESMF_STAGGERLOC_CORNER, & - farrayPtr=corner_lons, rc=status) - call ESMF_GridGetCoord(grid,localDE=0,coordDim=2,staggerloc=ESMF_STAGGERLOC_CORNER, & - farrayPtr=corner_lats, rc=status) + !call ESMF_GridGetCoord(grid,localDE=0,coordDim=1,staggerloc=ESMF_STAGGERLOC_CORNER, & + ! farrayPtr=corner_lons, _RC) + !call ESMF_GridGetCoord(grid,localDE=0,coordDim=2,staggerloc=ESMF_STAGGERLOC_CORNER, & + ! farrayPtr=corner_lats, _RC) + allocate(corner_lons(local_dims(1)+1, local_dims(2)+1)) + allocate(corner_lats(local_dims(1)+1, local_dims(2)+1)) + call MAPL_GridGetCorners(grid, corner_lons, corner_lats, _RC) if ( I1 == 1 .and. J1 == 1 ) then - allocate(lonRe(j2-j1+1), latRe(j2-j1+1)) - call MAPL_Reverse_Schmidt(grid, stretched, J2-J1+1, lonR8=corner_lons(1,:), & - latR8=corner_lats(1,:), lonRe=lonRe, latRe=latRe, _RC) + allocate(lonRe(local_dims(2)), latRe(local_dims(2))) + call MAPL_Reverse_Schmidt(grid, stretched, local_dims(2), lonR8=corner_lons(1,1:local_dims(2)), & + latR8=corner_lats(1,1:local_dims(2)), lonRe=lonRe, latRe=latRe, _RC) - allocate(accurate_lon(j2-j1+1), accurate_lat(j2-j1+1)) + allocate(accurate_lon(local_dims(2)), accurate_lat(local_dims(2))) shift0 = shift if (stretched) shift0 = 0 diff --git a/base/MaplGrid.F90 b/base/MaplGrid.F90 index fdac6371357e..153fdd11d865 100644 --- a/base/MaplGrid.F90 +++ b/base/MaplGrid.F90 @@ -265,10 +265,19 @@ subroutine MAPL_GridGet(GRID, globalCellCountPerDim, localCellCountPerDim, layou type(ESMF_DistGrid) :: distGrid integer, allocatable :: maxindex(:,:),minindex(:,:) integer, pointer :: ims(:),jms(:) + integer :: global_grid_info(10) pglobal = present(globalCellCountPerDim) plocal = present(localCellCountPerDim) + call ESMF_AttributeGet(grid, name="GLOBAL_GRID_INFO", isPresent=isPresent, _RC) + if (isPresent) then + call ESMF_AttributeGet(grid, name="GLOBAL_GRID_INFO", valueList=global_grid_info, _RC) + if (pglobal) globalCellCountPerDim = global_grid_info(1:3) + if (plocal) localCellCountPerDim = global_grid_info(4:6) + _RETURN(_SUCCESS) + end if + if (pglobal .or. plocal) then call ESMF_GridGet(grid, dimCount=gridRank, _RC) From 08bd7473eb848a8f4d92da8730e03421f56fa09f Mon Sep 17 00:00:00 2001 From: Amidu Oloso Date: Fri, 17 Jan 2025 12:09:37 -0500 Subject: [PATCH 3/4] removed unnecessary comments --- base/Base/Base_Base_implementation.F90 | 2 -- generic/OpenMP_Support.F90 | 1 - 2 files changed, 3 deletions(-) diff --git a/base/Base/Base_Base_implementation.F90 b/base/Base/Base_Base_implementation.F90 index 6653bf2e38fa..c04decee1501 100644 --- a/base/Base/Base_Base_implementation.F90 +++ b/base/Base/Base_Base_implementation.F90 @@ -2659,8 +2659,6 @@ module subroutine MAPL_GetHorzIJIndex(npts,II,JJ,lon,lat,lonR8,latR8,Grid, rc) end if if (im_world*6==jm_world) then -! call ESMF_AttributeGet(grid, name='GridType', value=grid_type, _RC) -! if(trim(grid_type) == "Cubed-Sphere") then call MAPL_GetGlobalHorzIJIndex(npts, II, JJ, lon=lon, lat=lat, lonR8=lonR8, latR8=latR8, Grid=Grid, _RC) diff --git a/generic/OpenMP_Support.F90 b/generic/OpenMP_Support.F90 index 661b5b0ace9a..9a9c33b629b1 100644 --- a/generic/OpenMP_Support.F90 +++ b/generic/OpenMP_Support.F90 @@ -189,7 +189,6 @@ function make_subgrids_from_bounds(primary_grid, bounds, unusable, rc) result(su global_grid_info(8) = i2 global_grid_info(9) = j1 + bounds(i)%min - 1 global_grid_info(10) = j1 + bounds(i)%max - 1 - print '(a,i6,6i4)', __FILE__, __LINE__, myPet, i, j1, j2, global_grid_info(9), global_grid_info(10) call ESMF_AttributeSet(subgrids(i), name="GLOBAL_GRID_INFO", & itemCount=10, valueList=global_grid_info, _RC) end block From 260d563d08ff0d3e8436ec33ed98a362110e731b Mon Sep 17 00:00:00 2001 From: Matthew Thompson Date: Thu, 23 Jan 2025 13:35:20 -0500 Subject: [PATCH 4/4] Update changelog --- CHANGELOG.md | 3 +++ 1 file changed, 3 insertions(+) diff --git a/CHANGELOG.md b/CHANGELOG.md index ef026ba9b20a..25e2e361ff82 100644 --- a/CHANGELOG.md +++ b/CHANGELOG.md @@ -10,10 +10,13 @@ and this project adheres to [Semantic Versioning](https://semver.org/spec/v2.0.0 ### Added ### Changed + - Updated ExtData so that if files are missing in a sequence the last value will be perisisted if one has not chosen `exact` option ### Fixed +- Changes were made to add attributes to the subgrids (i.e. created by dividing the MPI subdomain into smaller subdomains equal to the number of OpenMP threads) such that the correct dimensions for the MPI subdomain could be retrieved from the subgrids where ever needed. + ### Removed ### Deprecated