From 9a52faa9558508586cdb412ab2fe6ffe7274e375 Mon Sep 17 00:00:00 2001 From: "ukmo-juan.castillo" Date: Mon, 9 Mar 2026 14:57:08 +0000 Subject: [PATCH 1/6] Create a new lfric2lfric partitioning for lbc meshes --- .../lfric-driver/HEAD/rose-meta.conf | 15 +- .../mesh/runtime_partition_lfric_mod.f90 | 5 + .../source/mesh/runtime_partition_mod.f90 | 28 +- .../source/mesh/global_mesh_mod.F90 | 6 +- .../source/mesh/panel_decomposition_mod.f90 | 108 +++ infrastructure/source/mesh/partition_mod.F90 | 802 ++++++++++++++---- .../lfric-mesh_tools/HEAD/rose-meta.conf | 8 +- 7 files changed, 807 insertions(+), 165 deletions(-) diff --git a/components/driver/rose-meta/lfric-driver/HEAD/rose-meta.conf b/components/driver/rose-meta/lfric-driver/HEAD/rose-meta.conf index 980ecd18b..b603e03d2 100644 --- a/components/driver/rose-meta/lfric-driver/HEAD/rose-meta.conf +++ b/components/driver/rose-meta/lfric-driver/HEAD/rose-meta.conf @@ -128,6 +128,7 @@ description=The geometry on which the domain is embedded !enumeration=true fail-if=this == "'spherical'" and namelist:base_mesh=topology == "'fully_periodic'" and namelist:partitioning=partitioner != "'cubedsphere'" and namelist:base_mesh=prepartitioned == ".false." ; =this == "'planar'" and namelist:partitioning=partitioner != "'planar'" and namelist:base_mesh=prepartitioned == ".false." ; + =this == "'planar'" and namelist:partitioning=partitioner != "'lfric2lfric_lbc'" and namelist:base_mesh=prepartitioned == ".false." ; help=Along with topology this describes the domain. The geometry is the shape =on which the domain is embedded. This is currently either 'spherical' or ='planar'. @@ -665,7 +666,8 @@ description=Global mesh panel partitioning. help=For parallel computing, the 2D global mesh is divided up into partitions. =Each process rank runs an instance of the model on one partition. The =partition decompostion is specified on a `per panel` basis. - =i.e. The cubedsphere has six panels; the planar mesh has one panel. + =i.e. The cubedsphere has six panels; the planar and lfric2lfric lbc meshes + =have one panel. ns=namelist/Model/Mesh/Partitioning sort-key=Section-A02 @@ -718,6 +720,7 @@ help=Partitioner will attempt to generate partitioned panels based = * custom: x/y decompositions explicitly requested using = namelist:partitioning=panel_xproc, = namelist:partitioning=panel_yproc. + = * lfric2lfric_lbc: Especial partitioning for lbc meshes = * auto_nonuniform: As auto but allow columns of partitions = of differing heights. = * guided_nonuniform: Partition into columns according to @@ -727,8 +730,8 @@ help=Partitioner will attempt to generate partitioned panels based sort-key=Panel-A01 trigger=namelist:partitioning=panel_xproc: this == "'custom'" or this == "'guided_nonuniform'" ; =namelist:partitioning=panel_yproc: this == "'custom'" ; -value-titles=Auto, Single row, Single column, Custom, Auto nonuniform, Guided nonuniform -values='auto', 'row', 'column', 'custom', 'auto_nonuniform', 'guided_nonuniform' +value-titles=Auto, Single row, Single column, Custom, Lfric2lfric boundaries, Auto nonuniform, Guided nonuniform +values='auto', 'row', 'column', 'custom', 'lfric2lfric_lbc', 'auto_nonuniform', 'guided_nonuniform' [namelist:partitioning=panel_xproc] compulsory=true @@ -762,11 +765,13 @@ description=?????? !enumeration=true fail-if=this == "'cubedsphere'" and not ( namelist:base_mesh=geometry == "'spherical'" and namelist:base_mesh=topology == "'fully_periodic'" ); =this == "'cubedsphere'" and not ( env=TOTAL_RANKS % 6 == 0 or env=TOTAL_RANKS == 1 ) ; + =this == "'lfric2lfric_lbc'" and not namelist:base_mesh=geometry=="'planar'" ; + =this == "'lfric2lfric_lbc'" and not namelist:partitioning=panel_decomposition="'lfric2lfric_lbc'" ; help=This should match the mesh being used. For planar domains or LAMs on the =sphere the partitioner should be set to "'planar'". sort-key=Panel-A04 -value-titles=Planar, Cubedsphere -values='planar', 'cubedsphere' +value-titles=Planar, Lfric2lfric boundaries, Cubedsphere +values='planar', 'lfric2lfric_lbc', 'cubedsphere' [namelist:partitioning=tile_size_x] compulsory=false diff --git a/components/driver/source/mesh/runtime_partition_lfric_mod.f90 b/components/driver/source/mesh/runtime_partition_lfric_mod.f90 index 155662def..375db3888 100644 --- a/components/driver/source/mesh/runtime_partition_lfric_mod.f90 +++ b/components/driver/source/mesh/runtime_partition_lfric_mod.f90 @@ -18,12 +18,14 @@ module runtime_partition_lfric_mod row_decomposition_type, & column_decomposition_type, & custom_decomposition_type, & + lfric2lfric_lbc_decomposition_type, & auto_nonuniform_decomposition_type, & guided_nonuniform_decomposition_type use partitioning_config_mod, only: panel_decomposition_auto, & panel_decomposition_row, & panel_decomposition_column, & panel_decomposition_custom, & + panel_decomposition_lfric2lfric_lbc, & panel_decomposition_auto_nonuniform, & panel_decomposition_guided_nonuniform @@ -111,6 +113,9 @@ subroutine get_partition_parameters_nml( partitioning, & call partitioning%get_value( 'panel_yproc', panel_yproc ) decomposition = custom_decomposition_type( panel_xproc, panel_yproc ) + case ( panel_decomposition_lfric2lfric_lbc ) + decomposition = lfric2lfric_lbc_decomposition_type() + case ( panel_decomposition_auto_nonuniform ) decomposition = auto_nonuniform_decomposition_type() diff --git a/components/driver/source/mesh/runtime_partition_mod.f90 b/components/driver/source/mesh/runtime_partition_mod.f90 index eec3ae86e..b609ad0da 100644 --- a/components/driver/source/mesh/runtime_partition_mod.f90 +++ b/components/driver/source/mesh/runtime_partition_mod.f90 @@ -20,9 +20,11 @@ module runtime_partition_mod partitioner_interface, & partitioner_cubedsphere_serial, & partitioner_cubedsphere, & - partitioner_planar + partitioner_planar, & + partitioner_lfric2lfric_lbc - use panel_decomposition_mod, only: panel_decomposition_type, & + use panel_decomposition_mod, only: panel_decomposition_type, & + lfric2lfric_lbc_decomposition_type, & calc_mapping_factor use sci_query_mod, only: is_lbc @@ -36,8 +38,9 @@ module runtime_partition_mod public :: create_local_mesh public :: create_local_mesh_maps - integer, public, parameter :: mesh_cubedsphere = 34 - integer, public, parameter :: mesh_planar = 28 + integer, public, parameter :: mesh_cubedsphere = 34 + integer, public, parameter :: mesh_planar = 28 + integer, public, parameter :: mesh_lfric2lfric_lbc = 22 contains @@ -94,6 +97,12 @@ subroutine get_partition_strategy( mesh_selection, total_ranks, partitioner_ptr call log_event( "Using planar mesh partitioner ", & log_level_debug ) + case (mesh_lfric2lfric_lbc) + + partitioner_ptr => partitioner_lfric2lfric_lbc + call log_event( "Using lfric2lfric lbc mesh partitioner ", & + log_level_debug ) + end select end subroutine get_partition_strategy @@ -142,6 +151,8 @@ subroutine create_local_mesh( mesh_names, & integer(i_def) :: local_mesh_id, i integer(i_def) :: mapping_factor + logical(l_def) :: lfric2lfric_lbc + do i=1, size(mesh_names) global_mesh_ptr => global_mesh_collection%get_global_mesh( mesh_names(i) ) @@ -160,7 +171,14 @@ subroutine create_local_mesh( mesh_names, & ! Create local_mesh call local_mesh%initialise( global_mesh_ptr, partition ) - if ( .not. is_lbc(local_mesh) ) then + select type (decomposition) + type is (lfric2lfric_lbc_decomposition_type) + lfric2lfric_lbc = .true. + class default + lfric2lfric_lbc = .false. + end select + + if ( .not. is_lbc(local_mesh) .or. lfric2lfric_lbc) then ! Make sure the local_mesh cell owner lookup is correct ! (Can only be done when the code is running on its full set of MPI tasks) call local_mesh%init_cell_owner() diff --git a/infrastructure/source/mesh/global_mesh_mod.F90 b/infrastructure/source/mesh/global_mesh_mod.F90 index 1dbcb74f0..f1e11a9c4 100644 --- a/infrastructure/source/mesh/global_mesh_mod.F90 +++ b/infrastructure/source/mesh/global_mesh_mod.F90 @@ -1096,8 +1096,10 @@ function get_cell_id( self, cell_number, & end do ! y_dist cells in the y-direction ! Since the direction may have changed we need to recompute - y_index = rotate(x_index) - if ( y_cells < 0 ) y_index = opposite(y_index) + if (check) then + y_index = rotate(x_index) + if ( y_cells < 0 ) y_index = opposite(y_index) + end if ! y_index and y_dist do i = 1,y_dist diff --git a/infrastructure/source/mesh/panel_decomposition_mod.f90 b/infrastructure/source/mesh/panel_decomposition_mod.f90 index 0c0545722..df8fcc7fe 100644 --- a/infrastructure/source/mesh/panel_decomposition_mod.f90 +++ b/infrastructure/source/mesh/panel_decomposition_mod.f90 @@ -56,6 +56,13 @@ module panel_decomposition_mod procedure, public :: get_nprocs => get_column_nprocs end type column_decomposition_type + !> @brief Decomposition for lfric2lfric lbc meshes + type, extends(panel_decomposition_type), public :: lfric2lfric_lbc_decomposition_type + contains + procedure, public :: get_partition => get_lfric2lfric_lbc_partition + procedure, public :: get_nprocs => get_lfric2lfric_lbc_nprocs + end type lfric2lfric_lbc_decomposition_type + !> @brief Decomposition that automatically generates a nonuniform decomposition type, extends(panel_decomposition_type), public :: auto_nonuniform_decomposition_type contains @@ -560,6 +567,107 @@ function get_column_nprocs(self) result(nprocs) end function get_column_nprocs + !> @brief Partition the panel into an automatically determined number of x and + !> y processes for lbc meshes in lfric2lfric jobs + !> @details The lbc mesh just contains a rim of cells of width rim_width, + !> and no cells in the interior of the regional mesh. + !> We will partition the lbc mesh in rows or columns of withd equal + !> to the rim width: there are two columns of length num_cells_y - + !> rim_width, and two columns of length num_cells_x - rim_width. + !> The cells are arranged in a line starting from the SW corner + !> and moving in anticlockwise order. + !> @param[in] relative_rank The number of this rank in the order of all + !> ranks on the panel + !> @param[in] panel_ranks The total number of ranks on the panel + !> @param[in] mapping_factor The ratio between this and coarsest mesh + !> @param[in] num_cells_x The panel's size in the x direction + !> @param[in] num_cells_y The panel's size in the y direction + !> @param[in] any_maps Whether there exist maps between meshes that + !> must having aligning partitions, .false. by + !> default in this partitioning strategy + !> @param[inout] partition_width The width of the lbc rim + !> @param[inout] partition_height The partition's size in the y direction + !> @param[inout] partition_x_pos Position along the rim starting in the + !> SW corner + !> @param[inout] partition_y_pos Length of partition cells along the rim + subroutine get_lfric2lfric_lbc_partition( self, & + relative_rank, & + panel_ranks, & + mapping_factor, & + num_cells_x, & + num_cells_y, & + any_maps, & + partition_width, & + partition_height, & + partition_x_pos, & + partition_y_pos ) + implicit none + + class(lfric2lfric_lbc_decomposition_type), intent(in) :: self + integer(i_def), intent(in) :: relative_rank, & + panel_ranks, & + mapping_factor, & + num_cells_x, & + num_cells_y + logical, intent(in) :: any_maps + integer(i_def), intent(inout) :: partition_width, & + partition_height, & + partition_x_pos, & + partition_y_pos + + integer(i_def) :: lbc_length + + call log_event("Using lfric2lfric lbc decomposition", LOG_LEVEL_INFO) + + ! Try to partition the lbc rim in segments of the same size + lbc_length = 2*(num_cells_x+num_cells_y-2*partition_width) + + if (panel_ranks > lbc_length) then + write(log_scratch_space, "(a,i0,a,i0)") & + " Too many ranks: ", panel_ranks, & + " to decompose lbc of length:", lbc_length + call log_event(log_scratch_space, LOG_LEVEL_ERROR) + end if + + ! length of cells within the partition + partition_y_pos = nint(real(lbc_length) / real(panel_ranks)) + + if (panel_ranks*partition_y_pos /= lbc_length) then + write(log_scratch_space, "(a,i0,a,i0)") & + " Length of lbc rim:", lbc_length, & + " not divisible by total ranks:", panel_ranks + call log_event(log_scratch_space, LOG_LEVEL_ERROR) + end if + + ! first cell position in the partition + partition_x_pos = (relative_rank - 1) * partition_y_pos + 1 + + write(log_scratch_space, "(a,i0,a,i0,a,i0,a,i0)") & + " partition_x_pos ", partition_x_pos, & + " partition length", partition_y_pos, & + " partition_height ", partition_width + call log_event(log_scratch_space, LOG_LEVEL_INFO) + + end subroutine get_lfric2lfric_lbc_partition + + !> @brief Get the number of processors in the x- and y-direction. + !! For this class the function is not needed and so only + !! returns default values + !> @result nprocs Number of processors (x-dir, y-dir) + function get_lfric2lfric_lbc_nprocs(self) result(nprocs) + use constants_mod, only: i_def + + class(lfric2lfric_lbc_decomposition_type), intent(in) :: self + + integer(i_def) :: nprocs(2) + + ! These values aren't needed for column decomposition + ! (and aren't available until get_column_partition has been called) + ! so just return something that won't break the code + nprocs(:) = (/ 1, 1 /) + + end function get_lfric2lfric_lbc_nprocs + !> @brief Partition the panel into an automatically determined number of ! columns of partitions of variable size. !> @param[in] relative_rank The number of this rank in the order of all diff --git a/infrastructure/source/mesh/partition_mod.F90 b/infrastructure/source/mesh/partition_mod.F90 index a9ff9097e..cc334b0e9 100644 --- a/infrastructure/source/mesh/partition_mod.F90 +++ b/infrastructure/source/mesh/partition_mod.F90 @@ -38,6 +38,7 @@ module partition_mod public :: partitioner_cubedsphere, & partitioner_planar, & partitioner_cubedsphere_serial, & + partitioner_lfric2lfric_lbc, & partitioner_interface type, public :: partition_type @@ -626,6 +627,216 @@ subroutine partitioner_cubedsphere_serial( global_mesh, & end subroutine partitioner_cubedsphere_serial + !------------------------------------------------------------------------------- + ! Helper routine that creates a linked-list of all cells known to the partition, + ! including halos. + ! This will be ordered as: + ! inner n, inner n-1 ... inner 1, edge, halo 1 ... halo n-1, halo n + !------------------------------------------------------------------------------- + subroutine get_known_cells_halos( global_mesh, & + known_cells, & + partition, & + max_stencil_depth, & + generate_inner_halos, & + partitioned_cells, & + num_inner, & + num_edge, & + num_halo, & + num_ghost ) + + use linked_list_int_mod, only : linked_list_int_type + use linked_list_mod, only : linked_list_type, & + linked_list_item_type, & + before + + implicit none + + type(global_mesh_type), pointer, intent(in) :: global_mesh ! A global mesh object + type(linked_list_type), intent(inout) :: known_cells ! a list of cells known to the partition + type(linked_list_type), intent(in) :: partition ! a list of all cells in the partition + integer(i_def), intent(in) :: max_stencil_depth ! The maximum depth of stencil that will be used + logical(l_def), intent(in) :: generate_inner_halos ! Flag to control the generation of inner halos + integer(i_def), allocatable, intent(inout) :: partitioned_cells( : ) ! Returned array that holds the global ids of + ! all cells in local partition + integer(i_def), intent(out) :: num_inner( : ) ! Number of cells that are inner halo cells. + integer(i_def), intent(out) :: num_edge ! Number of cells that are owned by the partition, + integer(i_def), intent(out) :: num_halo( : ) ! Number of cells that are halo cells. + integer(i_def), intent(out) :: num_ghost ! Number of cells that are ghost cells - surrounding, + + ! local variables + + ! Create linked lists + type(linked_list_item_type), pointer :: last ! location of the last added cell in the list of cells + type(linked_list_item_type), pointer :: start_subsect ! start position when looping over subsections of cells + type(linked_list_item_type), pointer :: insert_point ! where to insert in a list + type(linked_list_item_type), pointer :: loop ! temp ptr to loop through list + + integer(i_def) :: start_sort, end_sort ! range over which to sort cells + integer(i_def) :: depth ! counter over the halo depths + integer(i_def) :: orig_num_in_list ! number of cells in list before halos are added + + integer(i_def) :: num_apply + integer(i_def) :: i + + + nullify( last ) + nullify( start_subsect ) + nullify( insert_point ) + nullify( loop ) + + ! get the number of edge cells currently stored in the known_cells list + num_edge = known_cells%get_length() + + ! Add all cells from the halos (up to max_stencil_depth) that are in a + ! stencil around each of the owned cells, but are not part of the partition. + ! Also add a "ghost" halo (max_stencil_depth+1) - used later to work out + ! dof ownerships + start_subsect => known_cells%get_head() ! start at the beginning + ! of known_cells list + ! num cells to apply stencil to is the number of edge cells + num_apply=num_edge + ! get a pointer to the known_cells list + !known_cells_ptr => known_cells + ! insert point is end of current list + insert_point => known_cells%get_tail() + do depth = 1,max_stencil_depth +1 + ! update number of cells currently in known_cells + orig_num_in_list = known_cells%get_length() + last => known_cells%get_tail() ! point at tail of current known_cells list + call apply_stencil( global_mesh, & + known_cells, & ! the current list of known cells + start_subsect, & ! where we want to start applying stencil in known_cells + num_apply, & ! the number of items in known_cells to iterate over + insert_point=insert_point, & ! where to insert in list + exclude=partition ) ! exclude cells in partition + if(depth <= max_stencil_depth)then + num_halo(depth) = known_cells%get_length() - orig_num_in_list ! num halo cells at this depth + ! is the number we just added + ! if cells were added at this depth, reset start point to previous end of known_cells list + if (num_halo(depth) > 0) then + start_subsect => last%next + end if + ! reset insert point to current end of list + insert_point => known_cells%get_tail() + ! num cells to apply stencil to next is the number of cells just added to known_cells + num_apply = num_halo(depth) + else + ! num ghost cells is the number we just added to known_cells at max_stencil_depth + 1 + num_ghost = known_cells%get_length() - orig_num_in_list + end if + end do + + ! Add all cells from the inner halos (up to max_stencil_depth) that are in a + ! stencil around each of the owned cells, but are not part of the outer halos + if ( generate_inner_halos ) then + call log_event( 'Generating Inner Halos', LOG_LEVEL_DEBUG ) + ! Point to start of known_cells list + start_subsect => known_cells%get_head() + ! insert point is head of known cells list as we want to insert before it + insert_point => known_cells%get_head() + ! num cells to apply stencil to is the number of edge cells only (not halo cells) + num_apply=num_edge + do depth = 1,max_stencil_depth-1 + ! update number of cells currently in known_cells + orig_num_in_list = known_cells%get_length() + call apply_stencil( global_mesh, & + known_cells, & + start_subsect, & + num_apply, & + insert_point=insert_point, & ! where to insert in list + placement=before ) + ! num inner halo cells at this depth is the number we just added to known_cells + num_inner(depth) = known_cells%get_length() - orig_num_in_list + ! num cells to apply stencil to next is the number of cells just added to known_cells + num_apply = num_inner(depth) + ! reset start point to current head of known_cells list + start_subsect => known_cells%get_head() + ! and reset insert point also + insert_point => known_cells%get_head() + end do + !Cells of depth max_stencil_depth + num_inner(max_stencil_depth)=0 + else + ! Do not create any inner halos + num_inner(:) = 0 + end if + + ! Now check partition list for any cells not yet added. These must be inner halo + ! update number of cells currently in known_cells + orig_num_in_list = known_cells%get_length() + ! point at head of partition list + loop => partition%get_head() + ! update insert point to head of known_cells list + insert_point => known_cells%get_head() + if (partition%get_length() > 0) then + + do i = 1,partition%get_length() + + if(.not. (known_cells%item_exists(loop%payload%get_id()))) then + + call known_cells%insert_item( linked_list_int_type(loop%payload%get_id()), & + insert_point=insert_point, placement=before) + end if + ! update insert point to head of known_cells list + insert_point => known_cells%get_head() + loop => loop%next + end do + ! update num_inner cells added + num_inner(max_stencil_depth)=known_cells%get_length() - orig_num_in_list + end if + + allocate(partitioned_cells(known_cells%get_length())) + ! reset loop to start of known_cells + loop => known_cells%get_head() + + + ! Copy cell ids from known_cells list to partitioned_cells array + do i = 1,known_cells%get_length() + partitioned_cells(i) = loop%payload%get_id() + if ( .not. associated(loop%next) ) exit !finished + loop => loop%next + end do + + + ! Deallocate the known_cells list + call known_cells%clear() + + ! Cell ids within the separate groups have to be in numerical order. + ! so (bubble) sort the separate groups + ! + ! Sort the individual depths of inner halo cells + end_sort=0 + do depth = max_stencil_depth, 1, -1 + start_sort = end_sort + 1 + end_sort = start_sort + num_inner(depth) - 1 + call bubble_sort( end_sort-start_sort+1, & + partitioned_cells(start_sort:end_sort) ) + end do + ! + ! Sort edge cells + start_sort = end_sort + 1 + end_sort = start_sort + num_edge - 1 + call bubble_sort( end_sort-start_sort+1, & + partitioned_cells(start_sort:end_sort) ) + ! + ! Sort the individual depths of halo cells + do depth = 1,max_stencil_depth + start_sort = end_sort + 1 + end_sort = start_sort + num_halo(depth) - 1 + call bubble_sort( end_sort-start_sort+1, & + partitioned_cells(start_sort:end_sort) ) + end do + ! + ! Sort the ghost halo + start_sort = end_sort + 1 + end_sort = start_sort + num_ghost - 1 + call bubble_sort( end_sort-start_sort+1, & + partitioned_cells(start_sort:end_sort) ) + + nullify( last, start_subsect, insert_point, loop ) + + end subroutine get_known_cells_halos + !------------------------------------------------------------------------------- ! Helper routine that partitions a mesh that is formed from a number ! of rectangular panels. @@ -645,9 +856,7 @@ subroutine partitioner_rectangular_panels( global_mesh, & num_ghost ) use linked_list_int_mod, only : linked_list_int_type - use linked_list_mod, only : linked_list_type, & - linked_list_item_type, & - before + use linked_list_mod, only : linked_list_type use reference_element_mod, only : W, S, E, N implicit none @@ -692,11 +901,6 @@ subroutine partitioner_rectangular_panels( global_mesh, & type(linked_list_type) :: partition ! a list of all cells in the partition type(linked_list_type), target :: known_cells ! a list of cells known to this partition - type(linked_list_item_type), pointer :: last ! location of the last added cell in the list of cells - type(linked_list_item_type), pointer :: start_subsect ! start position when looping over subsections of cells - type(linked_list_item_type), pointer :: insert_point ! where to insert in a list - type(linked_list_item_type), pointer :: loop ! temp ptr to loop through list - integer :: i, j ! loop counters integer :: cells(4) ! The cells around the vertex being queried integer :: oth1, oth2 ! When querying a cell around a vertex, these are @@ -709,9 +913,6 @@ subroutine partitioner_rectangular_panels( global_mesh, & integer :: cell_next_e ! The cell to the east of the cell being queried integer :: num_cells_x ! number of cells across a panel in x-direction integer :: num_cells_y ! number of cells across a panel in y-direction - integer :: start_sort, end_sort ! range over which to sort cells - integer :: depth ! counter over the halo depths - integer :: orig_num_in_list ! number of cells in list before halos are added integer :: ncell ! Number of cells integer :: cell_id ! Id of cell in the partition logical :: cross_panels ! Flag for partitioning across multiple cubed sphere panels @@ -722,7 +923,6 @@ subroutine partitioner_rectangular_panels( global_mesh, & integer :: nprocs(2) ! number of processors in the x- & y-direction integer :: xproc ! number of processsors in x-direction integer :: yproc ! number of processsors in y-direction - integer(i_def) :: num_apply logical(l_def) :: periodic_xy(2) ! Periodic in the x/y-axes periodic_xy = global_mesh%get_mesh_periodicity() @@ -731,11 +931,6 @@ subroutine partitioner_rectangular_panels( global_mesh, & n_cross_panels = 1 any_maps = global_mesh%get_nmaps() > 0 - nullify( last ) - nullify( start_subsect ) - nullify( insert_point ) - nullify( loop ) - nprocs = decomposition%get_nprocs() xproc = nprocs(1) yproc = nprocs(2) @@ -967,159 +1162,466 @@ subroutine partitioner_rectangular_panels( global_mesh, & ! Create a linked-list of all cells known to the partition, including halos. ! This will be ordered as: ! inner n, inner n-1 ... inner 1, edge, halo 1 ... halo n-1, halo n + call get_known_cells_halos( global_mesh, known_cells, partition, & + max_stencil_depth, generate_inner_halos, & + partitioned_cells, num_inner, & + num_edge, num_halo, num_ghost ) - ! get the number of edge cells currently stored in the known_cells list - num_edge = known_cells%get_length() + end subroutine partitioner_rectangular_panels - ! Add all cells from the halos (up to max_stencil_depth) that are in a - ! stencil around each of the owned cells, but are not part of the partition. - ! Also add a "ghost" halo (max_stencil_depth+1) - used later to work out - ! dof ownerships - start_subsect => known_cells%get_head() ! start at the beginning - ! of known_cells list - ! num cells to apply stencil to is the number of edge cells - num_apply=num_edge - ! get a pointer to the known_cells list - !known_cells_ptr => known_cells - ! insert point is end of current list - insert_point => known_cells%get_tail() - do depth = 1,max_stencil_depth +1 - ! update number of cells currently in known_cells - orig_num_in_list = known_cells%get_length() - last => known_cells%get_tail() ! point at tail of current known_cells list - call apply_stencil( global_mesh, & - known_cells, & ! the current list of known cells - start_subsect, & ! where we want to start applying stencil in known_cells - num_apply, & ! the number of items in known_cells to iterate over - insert_point=insert_point, & ! where to insert in list - exclude=partition ) ! exclude cells in partition - if(depth <= max_stencil_depth)then - num_halo(depth) = known_cells%get_length() - orig_num_in_list ! num halo cells at this depth - ! is the number we just added - ! if cells were added at this depth, reset start point to previous end of known_cells list - if (num_halo(depth) > 0) then - start_subsect => last%next - end if - ! reset insert point to current end of list - insert_point => known_cells%get_tail() - ! num cells to apply stencil to next is the number of cells just added to known_cells - num_apply = num_halo(depth) - else - ! num ghost cells is the number we just added to known_cells at max_stencil_depth + 1 - num_ghost = known_cells%get_length() - orig_num_in_list - end if - end do + !--------------------------------------------------------------------------- + !> @brief Partitions a lbc mesh on a plane for lfric2lfric. It returns + !> the global IDs of the cells in the given partition. + !> @details The lbc mesh just contains a rim of cells of width rim_width, + !> and no cells in the interior of the regional mesh. + !> We will partition the lbc mesh in rows or columns of withd equal + !> to the rim width: there are two columns of length num_cells_y - + !> rim_width, and two rows of length num_cells_x - rim_width. + !> The cells are arranged in a line starting from the SW corner + !> and moving in anticlockwise order. + !> + !> N + !> ||/////////// + !> || || + !> || || + !> W || || E + !> || || + !> ///////////|| + !> 0-> S + !> + !> @param[in] global_mesh A global mesh object that describes the layout + !> of the global mesh + !> @param[out] num_panels Number of panels in the 3D mesh: 1 by default + !> @param [in] decomposition Object containing decomposition parameters and + !> method + !> @param[in] local_rank Local MPI rank number. + !> @param[in] total_ranks Total number of MPI ranks. + !> @param[in] max_stencil_depth The maximum depth of stencil that will be + !> used with this partition. + !> @param[in] mapping_factor Ratio between this and coarsest associated + !> mesh + !> @param [in] generate_inner_halos Flag to control the generation of inner + !> halos + !> @param[inout] partitioned_cells Returned array that holds the global IDs + !> of all cells in local partition. + !> @param[out] num_inner Number of cells that are inner halo cells. + !> @param[out] num_edge Number of cells that are owned by the partition, + !> but may have dofs that are also owned by halo cells. + !> @param[out] num_halo Number of cells that are halo cells. + !> @param[out] num_ghost Number of "ghost" cells. These are cells in an + !> extra halo around the outermost actual halo and + !> are not in the partitioned domain, but are + !> required to fully describe the cells in the + !> partitioned domain. + !> + subroutine partitioner_lfric2lfric_lbc( global_mesh, & + num_panels, & + decomposition, & + local_rank, & + total_ranks, & + max_stencil_depth, & + mapping_factor, & + generate_inner_halos, & + partitioned_cells, & + num_inner, & + num_edge, & + num_halo, & + num_ghost ) - ! Add all cells from the inner halos (up to max_stencil_depth) that are in a - ! stencil around each of the owned cells, but are not part of the outer halos - if ( generate_inner_halos ) then - call log_event( 'Generating Inner Halos', LOG_LEVEL_DEBUG ) - ! Point to start of known_cells list - start_subsect => known_cells%get_head() - ! insert point is head of known cells list as we want to insert before it - insert_point => known_cells%get_head() - ! num cells to apply stencil to is the number of edge cells only (not halo cells) - num_apply=num_edge - do depth = 1,max_stencil_depth-1 - ! update number of cells currently in known_cells - orig_num_in_list = known_cells%get_length() - call apply_stencil( global_mesh, & - known_cells, & - start_subsect, & - num_apply, & - insert_point=insert_point, & ! where to insert in list - placement=before ) - ! num inner halo cells at this depth is the number we just added to known_cells - num_inner(depth) = known_cells%get_length() - orig_num_in_list - ! num cells to apply stencil to next is the number of cells just added to known_cells - num_apply = num_inner(depth) - ! reset start point to current head of known_cells list - start_subsect => known_cells%get_head() - ! and reset insert point also - insert_point => known_cells%get_head() + use linked_list_int_mod, only : linked_list_int_type + use linked_list_mod, only : linked_list_type, & + linked_list_item_type + use reference_element_mod, only : W, S, E, N + + implicit none + + type(global_mesh_type), pointer, intent(in) :: global_mesh ! A global mesh object + class(panel_decomposition_type), intent(in) :: decomposition ! An object specifying how to decompose the panel + + integer(i_def), intent(out) :: num_panels ! Number of panels that make up the mesh + integer(i_def), intent(in) :: local_rank ! Local MPI rank number + integer(i_def), intent(in) :: total_ranks ! Total number of MPI ranks + integer(i_def), intent(in) :: max_stencil_depth ! The maximum depth of stencil that will be used + ! with this partition + integer(i_def), intent(in) :: mapping_factor ! The ratio of number of edge cells between this and the + ! coarsest recursively mapped mesh + integer(i_def), allocatable, intent(inout) :: partitioned_cells( : ) ! Returned array that holds the global ids of + ! all cells in local partition + integer(i_def), intent(out) :: num_inner( : ) ! Number of cells that are inner halo cells. + integer(i_def), intent(out) :: num_edge ! Number of cells that are owned by the partition, + ! but may have dofs that are also owned by halo cells. + integer(i_def), intent(out) :: num_halo( : ) ! Number of cells that are halo cells. + integer(i_def), intent(out) :: num_ghost ! Number of cells that are ghost cells - surrounding, + ! but not in the partitioned domain + logical(l_def), intent(in) :: generate_inner_halos ! Flag to control the generation of inner halos + + integer(i_def) :: face ! which face of the cube is implied by local_rank (0->5) + integer(i_def) :: start_cell ! lowest cell id of the face implaced by local_rank + integer(i_def) :: start_rank ! The number of the first rank on the face implied by local_rank + integer(i_def) :: panel_ranks! The number of ranks per panel on the mesh + integer(i_def) :: relative_rank ! The position of the current rank relative to the first rank in its panel + integer(i_def) :: start_x ! global cell id of start of the domain on this partition in x-dirn + integer(i_def) :: num_x ! number of cells in the domain on this partition in x-dirn + integer(i_def) :: start_y ! global cell id of start of the domain on this partition in y-dirn + integer(i_def) :: num_y ! number of cells in the domain on this partition in y-dirn + integer(i_def) :: rim_width ! rim width of the lbc mesh + integer(i_def) :: ix, iy ! loop counters over cells on this partition in x- and y-dirns + integer(i_def) :: void_cell ! Cell id that marks the cell as a cell outside of the partition. + logical :: any_maps ! Whether there exist maps between meshes, meaning their partitions must align. + + ! Create linked lists + + type(linked_list_type) :: partition ! a list of all cells in the partition + type(linked_list_type), target :: known_cells ! a list of cells known to this partition + + integer, allocatable :: sw_corner_cells(:) + ! List of cells at the SW corner of the panels + integer :: cell ! starting point for num_cells_x calculation + integer :: cell_next(4) ! The cells around the cell being queried + integer :: cell_next_e ! The cell to the east of the cell being queried + integer :: cell_next_n ! The cell to the north of the cell being queried + integer :: num_cells_x ! number of cells across a panel in x-direction + integer :: num_cells_y ! number of cells across a panel in y-direction + integer :: ncells + logical(l_def) :: periodic_xy(2) ! Periodic in the x/y-axes + + periodic_xy = global_mesh%get_mesh_periodicity() + void_cell = global_mesh%get_void_cell() + any_maps = global_mesh%get_nmaps() > 0 + num_panels = 1 + + ! A single panelled mesh might be rectangular - so find the dimensions + ! First determine the southwest corner cell depending on periodicity. If + ! biperiodic, cell ID 1 can be used as mesh conectivity loops round + cell = 1 + call global_mesh%get_cell_next(cell,cell_next) + if ( .not. periodic_xy(1) ) then + ! If not periodic in E-W direction then walk West until you reach mesh + ! edge defined by the void cell. + do while (cell_next(W) /= void_cell) + cell = cell_next(W) + call global_mesh%get_cell_next(cell,cell_next) + end do + end if + + if ( .not. periodic_xy(2) ) then + ! If not periodic in N-S direction then walk South until you reach mesh + ! edge defined by the void cell. + do while (cell_next(S) /= void_cell) + cell = cell_next(S) + call global_mesh%get_cell_next(cell,cell_next) end do - !Cells of depth max_stencil_depth - num_inner(max_stencil_depth)=0 else - ! Do not create any inner halos - num_inner(:) = 0 + ! To make multigrid mesh partitions line up correctly in the periodic + ! case, the SW corners of all mesh domains need to be lined up. The + ! mesh generator currently generates meshes with cell no.1 at the NW + ! corner, so find the SW corner by moving one cell north from cell 1 + ! (across the wrap-around) + cell = 1 + call global_mesh%get_cell_next(cell,cell_next) + cell = cell_next(N) end if - ! Now check partition list for any cells not yet added. These must be inner halo - ! update number of cells currently in known_cells - orig_num_in_list = known_cells%get_length() - ! point at head of partition list - loop => partition%get_head() - ! update insert point to head of known_cells list - insert_point => known_cells%get_head() - if (partition%get_length() > 0) then + ! Assign cell ID of SW corner of mesh + allocate(sw_corner_cells(1)) + sw_corner_cells(1) = cell + + ! Work out number of cells in the x direction + num_cells_x = 1 + + ! Starting in the SW corner of the mesh so must walk East on non-periodic + ! meshes to determine number of cells in the x direction + ncells=global_mesh%get_ncells() + call global_mesh%get_cell_next(sw_corner_cells(1),cell_next) + cell_next_e = cell_next(E) + do while (cell_next_e /= sw_corner_cells(1) .and. cell_next_e /= void_cell) + num_cells_x=num_cells_x+1 + call global_mesh%get_cell_next(cell_next_e, cell_next) + cell_next_e = cell_next(E) + end do - do i = 1,partition%get_length() + ! Work out number of cells in the y direction + num_cells_y = 1 + + ! Starting in the SW corner of the mesh so must walk North on non-periodic + ! meshes to determine number of cells in the y direction + call global_mesh%get_cell_next(sw_corner_cells(1),cell_next) + cell_next_n = cell_next(N) + do while (cell_next_n /= sw_corner_cells(1) .and. cell_next_n /= void_cell) + num_cells_y=num_cells_y+1 + call global_mesh%get_cell_next(cell_next_n, cell_next) + cell_next_n = cell_next(N) + end do - if(.not. (known_cells%item_exists(loop%payload%get_id()))) then + ! Calculate rim width: solution of a quadratic equation + rim_width = nint(0.25*((num_cells_x+num_cells_y) - & + sqrt((num_cells_x+num_cells_y)*(num_cells_x+num_cells_y) - & + 4.0*ncells))) - call known_cells%insert_item( linked_list_int_type(loop%payload%get_id()), & - insert_point=insert_point, placement=before) - end if - ! update insert point to head of known_cells list - insert_point => known_cells%get_head() - loop => loop%next + face = (local_rank / total_ranks) + 1 + panel_ranks = total_ranks + start_rank = panel_ranks * (face - 1) + relative_rank = local_rank - start_rank + 1 + + call decomposition%get_partition( relative_rank, & + panel_ranks, & + mapping_factor, & + num_cells_x, & + num_cells_y, & + any_maps, & + rim_width, & + num_y, & + start_x, & + start_y ) + + + start_cell = sw_corner_cells(face) + deallocate(sw_corner_cells) + + ! Create a linked list of all cells that are part of this partition (not halos) + partition = linked_list_type() + + ! Find also those cells owned by the partition - but are on + ! the edge of the partitioned domain, so may have dofs shared with halo cells + known_cells = linked_list_type() + + ! South cells + if (start_x <= num_cells_x - rim_width) then + ! if processing more number of cells than are in the + ! bottom row, process only the bottom row for the moment + if (start_x + start_y > num_cells_x - rim_width) then + num_y = num_cells_x - rim_width - 1 + else + num_y = start_x + start_y - 1 + end if + do iy = 0, rim_width-1 + do ix = start_x-1, num_y + call partition%insert_item( linked_list_int_type( & + global_mesh%get_cell_id(start_cell, ix, iy))) + end do end do - ! update num_inner cells added - num_inner(max_stencil_depth)=known_cells%get_length() - orig_num_in_list + + ! left edge of the domain + do iy = 0, rim_width-1 + call known_cells%insert_item(linked_list_int_type( & + global_mesh%get_cell_id(start_cell, start_x-1, iy))) + end do + ! right edge of the domain + if (start_x + start_y <= num_cells_x - rim_width .and. & + start_x-1 /= num_y) then + do iy = 0, rim_width-1 + call known_cells%insert_item(linked_list_int_type( & + global_mesh%get_cell_id(start_cell, num_y, iy))) + end do + end if end if - allocate(partitioned_cells(known_cells%get_length())) - ! reset loop to start of known_cells - loop => known_cells%get_head() + ! East cells + if ( (start_x > num_cells_x - rim_width .and. & + start_x + start_y <= num_cells_x + num_cells_y - 2*rim_width) .or. & + start_x + start_y > num_cells_x + num_cells_y - 2*rim_width ) then + ! if processing more number of cells that are in the East column, + ! process only the East column for the moment + if (start_x + start_y > num_cells_x + num_cells_y - 2*rim_width) then + num_y = num_cells_y - rim_width - 1 + else + num_y = start_x + start_y - num_cells_x + rim_width + end if + if (start_x <= num_cells_x - rim_width) then + num_x = 0 + else + num_x = start_x - num_cells_x + rim_width - 1 + end if + do iy = num_x, num_y + do ix = num_cells_x-rim_width, num_cells_x-1 + call partition%insert_item( linked_list_int_type( & + global_mesh%get_cell_id(start_cell, ix, iy))) + end do + end do + ! bottom edge of the domain + if (start_x > num_cells_x - rim_width) then + if (start_x >= num_cells_x) then + do ix = num_cells_x-rim_width, num_cells_x-1 + call known_cells%insert_item(linked_list_int_type( & + global_mesh%get_cell_id(start_cell, ix, num_x))) + end do + else if (start_x == num_cells_x - rim_width + 1) then + ! only a left edge + do iy = 0, rim_width -1 + call known_cells%insert_item(linked_list_int_type( & + global_mesh%get_cell_id(start_cell, start_x-1, iy))) + end do + else + ! the edge has a L-shape: + ! left edge + do iy = num_x+1, rim_width -1 + call known_cells%insert_item(linked_list_int_type( & + global_mesh%get_cell_id(start_cell, num_cells_x-rim_width, iy))) + end do + ! bottom edge + do ix = num_cells_x-rim_width, num_cells_x-1 + call known_cells%insert_item(linked_list_int_type( & + global_mesh%get_cell_id(start_cell, ix, num_x))) + end do + end if + end if + ! top edge of the domain + if (start_x + start_y <= num_cells_x + num_cells_y - 2*rim_width .and. & + num_x /= num_y) then + do ix = num_cells_x-rim_width, num_cells_x-1 + call known_cells%insert_item(linked_list_int_type( & + global_mesh%get_cell_id(start_cell, ix, num_y))) + end do + end if - ! Copy cell ids from known_cells list to partitioned_cells array - do i = 1,known_cells%get_length() - partitioned_cells(i) = loop%payload%get_id() - if ( .not. associated(loop%next) ) exit !finished - loop => loop%next - end do + end if + ! Because a lbc mesh is empty in the middle, to deal with north and west + ! cells we will start advancing from the NE cell, which we have to find + cell = 1 + call global_mesh%get_cell_next(cell,cell_next) + if ( .not. periodic_xy(1) ) then + ! If not periodic in E-W direction then walk East until you reach mesh + ! edge defined by the void cell. + do while (cell_next(E) /= void_cell) + cell = cell_next(E) + call global_mesh%get_cell_next(cell,cell_next) + end do + end if - ! Deallocate the known_cells list - call known_cells%clear() + if ( .not. periodic_xy(2) ) then + ! If not periodic in N-S direction then walk North until you reach mesh + ! edge defined by the void cell. + do while (cell_next(N) /= void_cell) + cell = cell_next(N) + call global_mesh%get_cell_next(cell,cell_next) + end do + end if + start_cell = cell + + ! North cells + if ( (start_x > num_cells_x + num_cells_y - 2*rim_width .and. & + start_x + start_y <= 2*num_cells_x + num_cells_y - 3*rim_width) .or. & + start_x + start_y > 2*num_cells_x + num_cells_y - 3*rim_width ) then + if (start_x + start_y > 2*num_cells_x + num_cells_y - 3*rim_width) then + num_y = num_cells_y - rim_width - 1 + else + num_y = start_x + start_y - (num_cells_x + num_cells_y - 2*rim_width + 1) + end if + if (start_x <= num_cells_x + num_cells_y - 2*rim_width) then + num_x = 0 + else + num_x = start_x - (num_cells_x + num_cells_y - 2*rim_width + 1) + end if + do iy = 0, 1 - rim_width, -1 + do ix = -num_x, -num_y, -1 + call partition%insert_item( linked_list_int_type( & + global_mesh%get_cell_id(start_cell, ix, iy))) + end do + end do - ! Cell ids within the separate groups have to be in numerical order. - ! so (bubble) sort the separate groups - ! - ! Sort the individual depths of inner halo cells - end_sort=0 - do depth = max_stencil_depth, 1, -1 - start_sort = end_sort + 1 - end_sort = start_sort + num_inner(depth) - 1 - call bubble_sort( end_sort-start_sort+1, & - partitioned_cells(start_sort:end_sort) ) - end do - ! - ! Sort edge cells - start_sort = end_sort + 1 - end_sort = start_sort + num_edge - 1 - call bubble_sort( end_sort-start_sort+1, & - partitioned_cells(start_sort:end_sort) ) - ! - ! Sort the individual depths of halo cells - do depth = 1,max_stencil_depth - start_sort = end_sort + 1 - end_sort = start_sort + num_halo(depth) - 1 - call bubble_sort( end_sort-start_sort+1, & - partitioned_cells(start_sort:end_sort) ) - end do - ! - ! Sort the ghost halo - start_sort = end_sort + 1 - end_sort = start_sort + num_ghost - 1 - call bubble_sort( end_sort-start_sort+1, & - partitioned_cells(start_sort:end_sort) ) + ! right edge of the domain + if (start_x > num_cells_x + num_cells_y - 2*rim_width) then + if (start_x >= num_cells_x + num_cells_y - rim_width) then + do iy = 0, 1 - rim_width, -1 + call known_cells%insert_item(linked_list_int_type( & + global_mesh%get_cell_id(start_cell, -num_x, iy))) + end do + else if (start_x == num_cells_x + num_cells_y - 2*rim_width + 1) then + ! only a bottom edge + do ix = 0, 1 - rim_width, -1 + call known_cells%insert_item(linked_list_int_type( & + global_mesh%get_cell_id(start_cell, ix, 1-rim_width))) + end do + else + ! the edge has a L-shape: + ! bottom edge + do ix = 0, -num_x+1, -1 + call known_cells%insert_item(linked_list_int_type( & + global_mesh%get_cell_id(start_cell, ix, 1-rim_width))) + end do + ! right edge + do iy = 0, 1 - rim_width, -1 + call known_cells%insert_item(linked_list_int_type( & + global_mesh%get_cell_id(start_cell, -num_x, iy))) + end do + end if + end if + ! left edge of the domain + if (start_x + start_y <= 2*num_cells_x + num_cells_y - 3*rim_width .and. & + num_x /= num_y) then + do iy = 0, 1 - rim_width, -1 + call known_cells%insert_item(linked_list_int_type( & + global_mesh%get_cell_id(start_cell, -num_y, iy))) + end do + end if + end if - nullify( last, start_subsect, insert_point, loop ) + ! West cells + if ( start_x > 2*num_cells_x + num_cells_y - 3*rim_width .or. & + start_x + start_y > 2*num_cells_x + num_cells_y - 3*rim_width ) then + num_y = start_x + start_y - (2*num_cells_x + num_cells_y - 3*rim_width + 1) - 1 + if (start_x + start_y > 2*num_cells_x + num_cells_y - 3*rim_width) then + num_x = 0 + else + num_x = start_x - (2*num_cells_x + num_cells_y - 3*rim_width + 1) + end if + do iy = -num_x, -num_y, -1 + do ix = rim_width - num_cells_x, 1 - num_cells_x, -1 + call partition%insert_item( linked_list_int_type( & + global_mesh%get_cell_id(start_cell, ix, iy))) + end do + end do - end subroutine partitioner_rectangular_panels + ! top edge of the domain + if (start_x > 2*num_cells_x + num_cells_y - 3*rim_width) then + if (start_x >= 2*num_cells_x + num_cells_y - 2*rim_width) then + do ix = rim_width - num_cells_x, 1 - num_cells_x, -1 + call known_cells%insert_item(linked_list_int_type( & + global_mesh%get_cell_id(start_cell, ix, -num_x))) + end do + else if (start_x + start_y == 2*num_cells_x + num_cells_y - 3*rim_width) then + ! only a top edge + do ix = rim_width - num_cells_x, 1 - num_cells_x, -1 + call known_cells%insert_item(linked_list_int_type( & + global_mesh%get_cell_id(start_cell, ix, 0))) + end do + else + ! the edge has a L-shape: + ! right edge + do iy = 0, -num_x+1 + call known_cells%insert_item(linked_list_int_type( & + global_mesh%get_cell_id(start_cell, rim_width-num_cells_x, iy))) + end do + ! top edge + do ix = rim_width - num_cells_x, 1 - num_cells_x, -1 + call known_cells%insert_item(linked_list_int_type( & + global_mesh%get_cell_id(start_cell, ix, -num_x))) + end do + end if + end if + ! bottom edge of the domain + if (num_x /= num_y) then + do ix = rim_width - num_cells_x, 1 - num_cells_x, -1 + call known_cells%insert_item(linked_list_int_type( & + global_mesh%get_cell_id(start_cell, ix, -num_y))) + end do + end if + end if + + ! get the number of edge cells currently stored in the known_cells list + num_edge = known_cells%get_length() + + ! Create a linked-list of all cells known to the partition, including halos. + ! This will be ordered as: + ! inner n, inner n-1 ... inner 1, edge, halo 1 ... halo n-1, halo n + call get_known_cells_halos( global_mesh, known_cells, partition, & + max_stencil_depth, generate_inner_halos, & + partitioned_cells, num_inner, & + num_edge, num_halo, num_ghost ) + + end subroutine partitioner_lfric2lfric_lbc !--------------------------------------------------------------------------- ! Applies a stencil around a collection of cells. diff --git a/mesh_tools/rose-meta/lfric-mesh_tools/HEAD/rose-meta.conf b/mesh_tools/rose-meta/lfric-mesh_tools/HEAD/rose-meta.conf index 9f3b12dd5..f538bcf71 100644 --- a/mesh_tools/rose-meta/lfric-mesh_tools/HEAD/rose-meta.conf +++ b/mesh_tools/rose-meta/lfric-mesh_tools/HEAD/rose-meta.conf @@ -207,7 +207,8 @@ description=Global mesh partitioning. help=For parallel computing, the 2D global mesh is divided up into partitions. =Each process rank runs an instance of the model on one partition. The =partition decompostion is specified on a `per panel` basis. - =i.e. The cubedsphere has six panels; the planar mesh has one panel. + =i.e. The cubedsphere has six panels; the planar mesh and the lfric2lfric lbc + =meshes have one panel. ns=namelist/mesh/partitions sort-key=Section-A05 title=Partitions @@ -254,6 +255,7 @@ help=Partitioner will attempt to generate partitioned panels based = * custom: x/y decompositions explicitly requested using = namelist:partitions=panel_xproc, = namelist:partitions=panel_yproc. + = * lfric2lfric_lbc: Especial partitioning for lbc meshes = * auto_nonuniform: As auto but allow columns of partitions = of differing heights. = * guided_nonuniform: Partition into columns according to @@ -263,8 +265,8 @@ help=Partitioner will attempt to generate partitioned panels based sort-key=Panel-A02 trigger=namelist:partitions=panel_xproc: this == "'custom'" or this == "'guided_nonuniform'" ; =namelist:partitions=panel_yproc: this == "'custom'" ; -value-titles=Auto, Single row, Single column, Custom, Auto nonuniform, Guided nonuniform -values='auto', 'row', 'column', 'custom', 'auto_nonuniform', 'guided_nonuniform' +value-titles=Auto, Single row, Single column, Custom, Lfric2lfric boundaries, Auto nonuniform, Guided nonuniform +values='auto', 'row', 'column', 'custom', 'lfric2lfric_lbc', 'auto_nonuniform', 'guided_nonuniform' [namelist:partitions=panel_xproc] compulsory=true From 0b705e00b248b1f3dd0debea06021377138a01d5 Mon Sep 17 00:00:00 2001 From: "ukmo-juan.castillo" Date: Mon, 9 Mar 2026 15:07:12 +0000 Subject: [PATCH 2/6] Add update macros --- components/driver/rose-meta/lfric-driver/versions.py | 10 ++++++++++ mesh_tools/rose-meta/lfric-mesh_tools/versions.py | 10 ++++++++++ 2 files changed, 20 insertions(+) diff --git a/components/driver/rose-meta/lfric-driver/versions.py b/components/driver/rose-meta/lfric-driver/versions.py index 01798ad2b..7877d0e75 100644 --- a/components/driver/rose-meta/lfric-driver/versions.py +++ b/components/driver/rose-meta/lfric-driver/versions.py @@ -31,3 +31,13 @@ def upgrade(self, config, meta_config=None): # Add settings return config, self.reports """ + +class vn31_t301(MacroUpgrade): + """Upgrade macro for ticket 301 by Juan M Castillo.""" + + BEFORE_TAG = "vn3.1" + AFTER_TAG = "vn3.1_t301" + + def upgrade(self, config, meta_config=None): + # Blank Upgrade Macro + return config, self.reports diff --git a/mesh_tools/rose-meta/lfric-mesh_tools/versions.py b/mesh_tools/rose-meta/lfric-mesh_tools/versions.py index 01798ad2b..7877d0e75 100644 --- a/mesh_tools/rose-meta/lfric-mesh_tools/versions.py +++ b/mesh_tools/rose-meta/lfric-mesh_tools/versions.py @@ -31,3 +31,13 @@ def upgrade(self, config, meta_config=None): # Add settings return config, self.reports """ + +class vn31_t301(MacroUpgrade): + """Upgrade macro for ticket 301 by Juan M Castillo.""" + + BEFORE_TAG = "vn3.1" + AFTER_TAG = "vn3.1_t301" + + def upgrade(self, config, meta_config=None): + # Blank Upgrade Macro + return config, self.reports From 583e352f129324630ffe7571cef7c0d013c1d55e Mon Sep 17 00:00:00 2001 From: "ukmo-juan.castillo" Date: Thu, 12 Mar 2026 21:41:47 +0000 Subject: [PATCH 3/6] Tested fixes and improvements --- infrastructure/source/mesh/global_mesh_mod.F90 | 7 +++---- infrastructure/source/mesh/panel_decomposition_mod.f90 | 10 ++++++---- 2 files changed, 9 insertions(+), 8 deletions(-) diff --git a/infrastructure/source/mesh/global_mesh_mod.F90 b/infrastructure/source/mesh/global_mesh_mod.F90 index f1e11a9c4..d6324e2b3 100644 --- a/infrastructure/source/mesh/global_mesh_mod.F90 +++ b/infrastructure/source/mesh/global_mesh_mod.F90 @@ -1096,10 +1096,9 @@ function get_cell_id( self, cell_number, & end do ! y_dist cells in the y-direction ! Since the direction may have changed we need to recompute - if (check) then - y_index = rotate(x_index) - if ( y_cells < 0 ) y_index = opposite(y_index) - end if + y_index = rotate(x_index) + if ( x_cells < 0 ) y_index = opposite(y_index) + if ( y_cells < 0 ) y_index = opposite(y_index) ! y_index and y_dist do i = 1,y_dist diff --git a/infrastructure/source/mesh/panel_decomposition_mod.f90 b/infrastructure/source/mesh/panel_decomposition_mod.f90 index df8fcc7fe..f629e7180 100644 --- a/infrastructure/source/mesh/panel_decomposition_mod.f90 +++ b/infrastructure/source/mesh/panel_decomposition_mod.f90 @@ -632,11 +632,13 @@ subroutine get_lfric2lfric_lbc_partition( self, & ! length of cells within the partition partition_y_pos = nint(real(lbc_length) / real(panel_ranks)) + ! if the number of ranks does not divide exactly the length + ! of the lbc rim, assign the last rank a different size + ! to completely fill the lbc rim length if (panel_ranks*partition_y_pos /= lbc_length) then - write(log_scratch_space, "(a,i0,a,i0)") & - " Length of lbc rim:", lbc_length, & - " not divisible by total ranks:", panel_ranks - call log_event(log_scratch_space, LOG_LEVEL_ERROR) + if (relative_rank == panel_ranks) then + partition_y_pos = lbc_length - (panel_ranks-1)*partition_y_pos + end if end if ! first cell position in the partition From dbe426006b1bf3651973ade25a820c110349823c Mon Sep 17 00:00:00 2001 From: "ukmo-juan.castillo" Date: Fri, 13 Mar 2026 17:16:58 +0000 Subject: [PATCH 4/6] Simplify the number of namelist parameters needed to partition a lbc mesh in lfric2lfric - lfric2lfric will partition it with the default lfric2lfric lbc mesh partitioner --- .../lfric-driver/HEAD/rose-meta.conf | 12 ++--- .../driver/rose-meta/lfric-driver/versions.py | 10 ---- .../mesh/runtime_partition_lfric_mod.f90 | 53 ++++++++++--------- .../lfric-mesh_tools/HEAD/rose-meta.conf | 5 +- .../rose-meta/lfric-mesh_tools/versions.py | 10 ---- 5 files changed, 33 insertions(+), 57 deletions(-) diff --git a/components/driver/rose-meta/lfric-driver/HEAD/rose-meta.conf b/components/driver/rose-meta/lfric-driver/HEAD/rose-meta.conf index b603e03d2..c87447950 100644 --- a/components/driver/rose-meta/lfric-driver/HEAD/rose-meta.conf +++ b/components/driver/rose-meta/lfric-driver/HEAD/rose-meta.conf @@ -128,7 +128,6 @@ description=The geometry on which the domain is embedded !enumeration=true fail-if=this == "'spherical'" and namelist:base_mesh=topology == "'fully_periodic'" and namelist:partitioning=partitioner != "'cubedsphere'" and namelist:base_mesh=prepartitioned == ".false." ; =this == "'planar'" and namelist:partitioning=partitioner != "'planar'" and namelist:base_mesh=prepartitioned == ".false." ; - =this == "'planar'" and namelist:partitioning=partitioner != "'lfric2lfric_lbc'" and namelist:base_mesh=prepartitioned == ".false." ; help=Along with topology this describes the domain. The geometry is the shape =on which the domain is embedded. This is currently either 'spherical' or ='planar'. @@ -720,7 +719,6 @@ help=Partitioner will attempt to generate partitioned panels based = * custom: x/y decompositions explicitly requested using = namelist:partitioning=panel_xproc, = namelist:partitioning=panel_yproc. - = * lfric2lfric_lbc: Especial partitioning for lbc meshes = * auto_nonuniform: As auto but allow columns of partitions = of differing heights. = * guided_nonuniform: Partition into columns according to @@ -730,8 +728,8 @@ help=Partitioner will attempt to generate partitioned panels based sort-key=Panel-A01 trigger=namelist:partitioning=panel_xproc: this == "'custom'" or this == "'guided_nonuniform'" ; =namelist:partitioning=panel_yproc: this == "'custom'" ; -value-titles=Auto, Single row, Single column, Custom, Lfric2lfric boundaries, Auto nonuniform, Guided nonuniform -values='auto', 'row', 'column', 'custom', 'lfric2lfric_lbc', 'auto_nonuniform', 'guided_nonuniform' +value-titles=Auto, Single row, Single column, Custom, Auto nonuniform, Guided nonuniform +values='auto', 'row', 'column', 'custom', 'auto_nonuniform', 'guided_nonuniform' [namelist:partitioning=panel_xproc] compulsory=true @@ -765,13 +763,11 @@ description=?????? !enumeration=true fail-if=this == "'cubedsphere'" and not ( namelist:base_mesh=geometry == "'spherical'" and namelist:base_mesh=topology == "'fully_periodic'" ); =this == "'cubedsphere'" and not ( env=TOTAL_RANKS % 6 == 0 or env=TOTAL_RANKS == 1 ) ; - =this == "'lfric2lfric_lbc'" and not namelist:base_mesh=geometry=="'planar'" ; - =this == "'lfric2lfric_lbc'" and not namelist:partitioning=panel_decomposition="'lfric2lfric_lbc'" ; help=This should match the mesh being used. For planar domains or LAMs on the =sphere the partitioner should be set to "'planar'". sort-key=Panel-A04 -value-titles=Planar, Lfric2lfric boundaries, Cubedsphere -values='planar', 'lfric2lfric_lbc', 'cubedsphere' +value-titles=Planar, Cubedsphere +values='planar', 'cubedsphere' [namelist:partitioning=tile_size_x] compulsory=false diff --git a/components/driver/rose-meta/lfric-driver/versions.py b/components/driver/rose-meta/lfric-driver/versions.py index 7877d0e75..01798ad2b 100644 --- a/components/driver/rose-meta/lfric-driver/versions.py +++ b/components/driver/rose-meta/lfric-driver/versions.py @@ -31,13 +31,3 @@ def upgrade(self, config, meta_config=None): # Add settings return config, self.reports """ - -class vn31_t301(MacroUpgrade): - """Upgrade macro for ticket 301 by Juan M Castillo.""" - - BEFORE_TAG = "vn3.1" - AFTER_TAG = "vn3.1_t301" - - def upgrade(self, config, meta_config=None): - # Blank Upgrade Macro - return config, self.reports diff --git a/components/driver/source/mesh/runtime_partition_lfric_mod.f90 b/components/driver/source/mesh/runtime_partition_lfric_mod.f90 index 375db3888..3351694c6 100644 --- a/components/driver/source/mesh/runtime_partition_lfric_mod.f90 +++ b/components/driver/source/mesh/runtime_partition_lfric_mod.f90 @@ -12,7 +12,8 @@ module runtime_partition_lfric_mod use namelist_collection_mod, only: namelist_collection_type use namelist_mod, only: namelist_type use partition_mod, only: partitioner_interface - use runtime_partition_mod, only: get_partition_strategy + use runtime_partition_mod, only: get_partition_strategy, & + mesh_lfric2lfric_lbc use panel_decomposition_mod, only: panel_decomposition_type, & auto_decomposition_type, & row_decomposition_type, & @@ -25,7 +26,6 @@ module runtime_partition_lfric_mod panel_decomposition_row, & panel_decomposition_column, & panel_decomposition_custom, & - panel_decomposition_lfric2lfric_lbc, & panel_decomposition_auto_nonuniform, & panel_decomposition_guided_nonuniform @@ -95,40 +95,41 @@ subroutine get_partition_parameters_nml( partitioning, & integer :: panel_decomposition - call partitioning%get_value( 'panel_decomposition', panel_decomposition ) - - select case (panel_decomposition) + if (mesh_selection == mesh_lfric2lfric_lbc) then + decomposition = lfric2lfric_lbc_decomposition_type() + else + call partitioning%get_value( 'panel_decomposition', panel_decomposition ) - case ( panel_decomposition_auto ) - decomposition = auto_decomposition_type() + select case (panel_decomposition) - case ( panel_decomposition_row ) - decomposition = row_decomposition_type() + case ( panel_decomposition_auto ) + decomposition = auto_decomposition_type() - case ( panel_decomposition_column ) - decomposition = column_decomposition_type() + case ( panel_decomposition_row ) + decomposition = row_decomposition_type() - case ( panel_decomposition_custom ) - call partitioning%get_value( 'panel_xproc', panel_xproc ) - call partitioning%get_value( 'panel_yproc', panel_yproc ) - decomposition = custom_decomposition_type( panel_xproc, panel_yproc ) + case ( panel_decomposition_column ) + decomposition = column_decomposition_type() - case ( panel_decomposition_lfric2lfric_lbc ) - decomposition = lfric2lfric_lbc_decomposition_type() + case ( panel_decomposition_custom ) + call partitioning%get_value( 'panel_xproc', panel_xproc ) + call partitioning%get_value( 'panel_yproc', panel_yproc ) + decomposition = custom_decomposition_type( panel_xproc, panel_yproc ) - case ( panel_decomposition_auto_nonuniform ) - decomposition = auto_nonuniform_decomposition_type() + case ( panel_decomposition_auto_nonuniform ) + decomposition = auto_nonuniform_decomposition_type() - case ( panel_decomposition_guided_nonuniform ) - call partitioning%get_value( 'panel_xproc', panel_xproc ) - decomposition = guided_nonuniform_decomposition_type( panel_xproc ) + case ( panel_decomposition_guided_nonuniform ) + call partitioning%get_value( 'panel_xproc', panel_xproc ) + decomposition = guided_nonuniform_decomposition_type( panel_xproc ) - case default - ! Not clear it's possible to still error at this point but no harm in checking - call log_event( "Missing entry for panel decomposition, "// & + case default + ! Not clear it's possible to still error at this point but no harm in checking + call log_event( "Missing entry for panel decomposition, "// & "specify 'auto' if unsure.", LOG_LEVEL_ERROR ) - end select + end select + end if call get_partition_strategy(mesh_selection, total_ranks, partitioner_ptr) diff --git a/mesh_tools/rose-meta/lfric-mesh_tools/HEAD/rose-meta.conf b/mesh_tools/rose-meta/lfric-mesh_tools/HEAD/rose-meta.conf index f538bcf71..153a7921f 100644 --- a/mesh_tools/rose-meta/lfric-mesh_tools/HEAD/rose-meta.conf +++ b/mesh_tools/rose-meta/lfric-mesh_tools/HEAD/rose-meta.conf @@ -255,7 +255,6 @@ help=Partitioner will attempt to generate partitioned panels based = * custom: x/y decompositions explicitly requested using = namelist:partitions=panel_xproc, = namelist:partitions=panel_yproc. - = * lfric2lfric_lbc: Especial partitioning for lbc meshes = * auto_nonuniform: As auto but allow columns of partitions = of differing heights. = * guided_nonuniform: Partition into columns according to @@ -265,8 +264,8 @@ help=Partitioner will attempt to generate partitioned panels based sort-key=Panel-A02 trigger=namelist:partitions=panel_xproc: this == "'custom'" or this == "'guided_nonuniform'" ; =namelist:partitions=panel_yproc: this == "'custom'" ; -value-titles=Auto, Single row, Single column, Custom, Lfric2lfric boundaries, Auto nonuniform, Guided nonuniform -values='auto', 'row', 'column', 'custom', 'lfric2lfric_lbc', 'auto_nonuniform', 'guided_nonuniform' +value-titles=Auto, Single row, Single column, Custom, Auto nonuniform, Guided nonuniform +values='auto', 'row', 'column', 'custom', 'auto_nonuniform', 'guided_nonuniform' [namelist:partitions=panel_xproc] compulsory=true diff --git a/mesh_tools/rose-meta/lfric-mesh_tools/versions.py b/mesh_tools/rose-meta/lfric-mesh_tools/versions.py index 7877d0e75..01798ad2b 100644 --- a/mesh_tools/rose-meta/lfric-mesh_tools/versions.py +++ b/mesh_tools/rose-meta/lfric-mesh_tools/versions.py @@ -31,13 +31,3 @@ def upgrade(self, config, meta_config=None): # Add settings return config, self.reports """ - -class vn31_t301(MacroUpgrade): - """Upgrade macro for ticket 301 by Juan M Castillo.""" - - BEFORE_TAG = "vn3.1" - AFTER_TAG = "vn3.1_t301" - - def upgrade(self, config, meta_config=None): - # Blank Upgrade Macro - return config, self.reports From 4f271ed371a9eb46254dd71050c3adfeee4ba0e2 Mon Sep 17 00:00:00 2001 From: "ukmo-juan.castillo" Date: Mon, 16 Mar 2026 08:24:25 +0000 Subject: [PATCH 5/6] Revert some of the changes to minimise changes to lfric_core - the code changes will be take care of in lfric apps --- .../mesh/runtime_partition_lfric_mod.f90 | 50 ++++++++----------- 1 file changed, 22 insertions(+), 28 deletions(-) diff --git a/components/driver/source/mesh/runtime_partition_lfric_mod.f90 b/components/driver/source/mesh/runtime_partition_lfric_mod.f90 index 3351694c6..155662def 100644 --- a/components/driver/source/mesh/runtime_partition_lfric_mod.f90 +++ b/components/driver/source/mesh/runtime_partition_lfric_mod.f90 @@ -12,14 +12,12 @@ module runtime_partition_lfric_mod use namelist_collection_mod, only: namelist_collection_type use namelist_mod, only: namelist_type use partition_mod, only: partitioner_interface - use runtime_partition_mod, only: get_partition_strategy, & - mesh_lfric2lfric_lbc + use runtime_partition_mod, only: get_partition_strategy use panel_decomposition_mod, only: panel_decomposition_type, & auto_decomposition_type, & row_decomposition_type, & column_decomposition_type, & custom_decomposition_type, & - lfric2lfric_lbc_decomposition_type, & auto_nonuniform_decomposition_type, & guided_nonuniform_decomposition_type use partitioning_config_mod, only: panel_decomposition_auto, & @@ -95,41 +93,37 @@ subroutine get_partition_parameters_nml( partitioning, & integer :: panel_decomposition - if (mesh_selection == mesh_lfric2lfric_lbc) then - decomposition = lfric2lfric_lbc_decomposition_type() - else - call partitioning%get_value( 'panel_decomposition', panel_decomposition ) + call partitioning%get_value( 'panel_decomposition', panel_decomposition ) - select case (panel_decomposition) + select case (panel_decomposition) - case ( panel_decomposition_auto ) - decomposition = auto_decomposition_type() + case ( panel_decomposition_auto ) + decomposition = auto_decomposition_type() - case ( panel_decomposition_row ) - decomposition = row_decomposition_type() + case ( panel_decomposition_row ) + decomposition = row_decomposition_type() - case ( panel_decomposition_column ) - decomposition = column_decomposition_type() + case ( panel_decomposition_column ) + decomposition = column_decomposition_type() - case ( panel_decomposition_custom ) - call partitioning%get_value( 'panel_xproc', panel_xproc ) - call partitioning%get_value( 'panel_yproc', panel_yproc ) - decomposition = custom_decomposition_type( panel_xproc, panel_yproc ) + case ( panel_decomposition_custom ) + call partitioning%get_value( 'panel_xproc', panel_xproc ) + call partitioning%get_value( 'panel_yproc', panel_yproc ) + decomposition = custom_decomposition_type( panel_xproc, panel_yproc ) - case ( panel_decomposition_auto_nonuniform ) - decomposition = auto_nonuniform_decomposition_type() + case ( panel_decomposition_auto_nonuniform ) + decomposition = auto_nonuniform_decomposition_type() - case ( panel_decomposition_guided_nonuniform ) - call partitioning%get_value( 'panel_xproc', panel_xproc ) - decomposition = guided_nonuniform_decomposition_type( panel_xproc ) + case ( panel_decomposition_guided_nonuniform ) + call partitioning%get_value( 'panel_xproc', panel_xproc ) + decomposition = guided_nonuniform_decomposition_type( panel_xproc ) - case default - ! Not clear it's possible to still error at this point but no harm in checking - call log_event( "Missing entry for panel decomposition, "// & + case default + ! Not clear it's possible to still error at this point but no harm in checking + call log_event( "Missing entry for panel decomposition, "// & "specify 'auto' if unsure.", LOG_LEVEL_ERROR ) - end select - end if + end select call get_partition_strategy(mesh_selection, total_ranks, partitioner_ptr) From bdd48a792ec043934767d354d8eea125e62b5e1a Mon Sep 17 00:00:00 2001 From: "ukmo-juan.castillo" Date: Mon, 16 Mar 2026 15:25:30 +0000 Subject: [PATCH 6/6] Small changes to metadata comments --- components/driver/rose-meta/lfric-driver/HEAD/rose-meta.conf | 2 +- mesh_tools/rose-meta/lfric-mesh_tools/HEAD/rose-meta.conf | 2 +- 2 files changed, 2 insertions(+), 2 deletions(-) diff --git a/components/driver/rose-meta/lfric-driver/HEAD/rose-meta.conf b/components/driver/rose-meta/lfric-driver/HEAD/rose-meta.conf index c87447950..18d0d363a 100644 --- a/components/driver/rose-meta/lfric-driver/HEAD/rose-meta.conf +++ b/components/driver/rose-meta/lfric-driver/HEAD/rose-meta.conf @@ -665,7 +665,7 @@ description=Global mesh panel partitioning. help=For parallel computing, the 2D global mesh is divided up into partitions. =Each process rank runs an instance of the model on one partition. The =partition decompostion is specified on a `per panel` basis. - =i.e. The cubedsphere has six panels; the planar and lfric2lfric lbc meshes + =i.e. The cubedsphere has six panels; the planar and lbc meshes =have one panel. ns=namelist/Model/Mesh/Partitioning sort-key=Section-A02 diff --git a/mesh_tools/rose-meta/lfric-mesh_tools/HEAD/rose-meta.conf b/mesh_tools/rose-meta/lfric-mesh_tools/HEAD/rose-meta.conf index 153a7921f..d136d938c 100644 --- a/mesh_tools/rose-meta/lfric-mesh_tools/HEAD/rose-meta.conf +++ b/mesh_tools/rose-meta/lfric-mesh_tools/HEAD/rose-meta.conf @@ -207,7 +207,7 @@ description=Global mesh partitioning. help=For parallel computing, the 2D global mesh is divided up into partitions. =Each process rank runs an instance of the model on one partition. The =partition decompostion is specified on a `per panel` basis. - =i.e. The cubedsphere has six panels; the planar mesh and the lfric2lfric lbc + =i.e. The cubedsphere has six panels; the planar mesh and the lbc =meshes have one panel. ns=namelist/mesh/partitions sort-key=Section-A05