From 118cf2287b785cb5c2119c4ecbd1f3d17ec95d1a Mon Sep 17 00:00:00 2001 From: "Alberto F. Martin" Date: Tue, 30 Jan 2024 21:49:25 +1100 Subject: [PATCH 01/29] * Massive refactoring of PXest methods to accommodate p6est * Started the implementation of AnisotropicallyAdapted3DDistributedDiscreteModels on top of p6est --- ...callyAdapted3DDistributedDiscreteModels.jl | 453 ++++ src/GridapP4est.jl | 3 + src/OctreeDistributedDiscreteModels.jl | 1419 +++--------- src/PXestTypeMethods.jl | 1927 +++++++++++++++++ ...mlyRefinedForestOfOctreesDiscreteModels.jl | 198 +- test.jl | 33 + test/PoissonNonConformingOctreeModelsTests.jl | 6 +- 7 files changed, 2725 insertions(+), 1314 deletions(-) create mode 100644 src/AnisotropicallyAdapted3DDistributedDiscreteModels.jl create mode 100644 src/PXestTypeMethods.jl create mode 100644 test.jl diff --git a/src/AnisotropicallyAdapted3DDistributedDiscreteModels.jl b/src/AnisotropicallyAdapted3DDistributedDiscreteModels.jl new file mode 100644 index 0000000..1352bdf --- /dev/null +++ b/src/AnisotropicallyAdapted3DDistributedDiscreteModels.jl @@ -0,0 +1,453 @@ +function AnisotropicallyAdapted3DDistributedDiscreteModel( + parts::AbstractVector{<:Integer}, + coarse_model::DiscreteModel{2,2}, + num_horizontal_uniform_refinements, + num_vertical_uniform_refinements; + extrusion_vector::Vector{Float64}=[0.0,0.0,1.0]) + + pXest_type=P6estType() + ptr_pXest_connectivity=setup_pXest_connectivity(pXest_type,coarse_model,extrusion_vector) + ptr_pXest=setup_pXest(pXest_type, + parts.comm, + ptr_pXest_connectivity, + num_horizontal_uniform_refinements, + num_vertical_uniform_refinements) + + + ptr_pXest_ghost=setup_pXest_ghost(pXest_type,ptr_pXest) + ptr_pXest_lnodes=setup_pXest_lnodes_nonconforming(pXest_type, ptr_pXest, ptr_pXest_ghost) + + dmodel,non_conforming_glue = setup_non_conforming_distributed_discrete_model(pXest_type, + parts, + coarse_model, + ptr_pXest_connectivity, + ptr_pXest, + ptr_pXest_ghost, + ptr_pXest_lnodes) + + pXest_lnodes_destroy(pXest_type,ptr_pXest_lnodes) + pXest_ghost_destroy(pXest_type,ptr_pXest_ghost) + + return OctreeDistributedDiscreteModel(3, + 3, + parts, + dmodel, + non_conforming_glue, + coarse_model, + ptr_pXest_connectivity, + ptr_pXest, + pXest_type, + true, + nothing) +end + +function _vertically_refine_coarsen_balance!(model::OctreeDistributedDiscreteModel{Dc,Dp}, + refinement_and_coarsening_flags::MPIArray{<:Vector}) where {Dc,Dp} + + pXest_type = model.pXest_type + init_fn_callback_c = p6est_vertically_adapt_reset_callbacks() + #coarsen_fn_callback_c = p6est_vertically_coarsen_callbacks() + refine_callback_c,refine_replace_callback_c = p6est_vertically_refine_callbacks() + + map(model.dmodel.models,refinement_and_coarsening_flags) do lmodel, flags + # The length of the local flags array has to match the number of + # cells in the model. This includes both owned and ghost cells. + # Only the flags for owned cells are actually taken into account. + @assert num_cells(lmodel)==length(flags) + pXest_reset_data!(pXest_type, model.ptr_pXest, Cint(sizeof(Cint)), init_fn_callback_c, pointer(flags)) + end + + # # Copy input p4est, refine and balance + ptr_new_pXest = pXest_copy(pXest_type, model.ptr_pXest) + p6est_vertically_refine!(ptr_new_pXest, + refine_callback_c, + refine_replace_callback_c) + #pXest_vertically_coarsen!(ptr_new_pXest, coarsen_fn_callback_c) + pXest_balance!(pXest_type, ptr_new_pXest) + p6est_vertically_adapt_update_flags!(model.ptr_pXest,ptr_new_pXest) + ptr_new_pXest +end + +function vertically_adapt(model::OctreeDistributedDiscreteModel{Dc,Dp}, + refinement_and_coarsening_flags::MPIArray{<:Vector{<:Integer}}; + parts=nothing) where {Dc,Dp} + + Gridap.Helpers.@notimplementedif parts!=nothing + + _refinement_and_coarsening_flags = map(refinement_and_coarsening_flags) do flags + convert(Vector{Cint},flags) + end + + ptr_new_pXest = _vertically_refine_coarsen_balance!(model, _refinement_and_coarsening_flags) + + # Extract ghost and lnodes + ptr_pXest_ghost = setup_pXest_ghost(model.pXest_type, ptr_new_pXest) + ptr_pXest_lnodes = setup_pXest_lnodes_nonconforming(model.pXest_type, ptr_new_pXest, ptr_pXest_ghost) + + # Build fine-grid mesh + fmodel,non_conforming_glue = setup_non_conforming_distributed_discrete_model(model.pXest_type, + model.parts, + model.coarse_model, + model.ptr_pXest_connectivity, + ptr_new_pXest, + ptr_pXest_ghost, + ptr_pXest_lnodes) + + pXest_ghost_destroy(model.pXest_type,ptr_pXest_ghost) + pXest_lnodes_destroy(model.pXest_type,ptr_pXest_lnodes) + adaptivity_glue = _compute_fine_to_coarse_model_glue(model.pXest_type, + PXestVerticalRefinementRuleType(), + model.parts, + model.dmodel, + fmodel, + _refinement_and_coarsening_flags) + adaptive_models = map(local_views(model), + local_views(fmodel), + adaptivity_glue) do model, fmodel, glue + Gridap.Adaptivity.AdaptedDiscreteModel(fmodel,model,glue) + end + fmodel = GridapDistributed.GenericDistributedDiscreteModel(adaptive_models,get_cell_gids(fmodel)) + ref_model = OctreeDistributedDiscreteModel(Dc,Dp, + model.parts, + fmodel, + non_conforming_glue, + model.coarse_model, + model.ptr_pXest_connectivity, + ptr_new_pXest, + model.pXest_type, + false, + model) + return ref_model, adaptivity_glue +end + +function setup_non_conforming_distributed_discrete_model(pXest_type::P6estType, + parts, + coarse_discrete_model, + ptr_pXest_connectivity, + ptr_pXest, + ptr_pXest_ghost, + ptr_pXest_lnodes) + + Dc=num_cell_dims(pXest_type) + + cell_prange = setup_cell_prange(pXest_type, parts, ptr_pXest, ptr_pXest_ghost) + + gridap_cell_faces, + non_conforming_glue= + generate_cell_faces_and_non_conforming_glue(pXest_type,ptr_pXest_lnodes, cell_prange) + + + nlvertices = map(non_conforming_glue) do ncglue + ncglue.num_regular_faces[1]+ncglue.num_hanging_faces[1] + end + + node_coordinates=generate_node_coordinates(pXest_type, + gridap_cell_faces[1], + nlvertices, + ptr_pXest_connectivity, + ptr_pXest, + ptr_pXest_ghost) + + grid,topology=generate_grid_and_topology(pXest_type, + gridap_cell_faces[1], + nlvertices, + node_coordinates) + + map(topology,gridap_cell_faces[Dc]) do topology,cell_faces + cell_faces_gridap = Gridap.Arrays.Table(cell_faces.data,cell_faces.ptrs) + topology.n_m_to_nface_to_mfaces[Dc+1,Dc] = cell_faces_gridap + topology.n_m_to_nface_to_mfaces[Dc,Dc+1] = Gridap.Geometry.generate_cells_around(cell_faces_gridap) + end + + if (Dc==3) + map(topology,gridap_cell_faces[Dc-1]) do topology,cell_edges + cell_edges_gridap = Gridap.Arrays.Table(cell_edges.data,cell_edges.ptrs) + topology.n_m_to_nface_to_mfaces[Dc+1,Dc-1] = cell_edges_gridap + topology.n_m_to_nface_to_mfaces[Dc-1,Dc+1] = Gridap.Geometry.generate_cells_around(cell_edges_gridap) + end + end + + face_labeling=generate_face_labeling(pXest_type, + parts, + cell_prange, + coarse_discrete_model, + topology, + ptr_pXest, + ptr_pXest_ghost) + + # _set_hanging_labels!(face_labeling,non_conforming_glue) + + discretemodel=map(grid,topology,face_labeling) do grid, topology, face_labeling + Gridap.Geometry.UnstructuredDiscreteModel(grid,topology,face_labeling) + end + GridapDistributed.DistributedDiscreteModel(discretemodel,cell_prange), non_conforming_glue +end + + +# Extrude entity Ids: +# 1. Extrude corners to edges and corners +const corner_to_extruded_corner=[2,4,6,8] +const corner_to_extruded_edge=[1,2,3,4] +# 2. Extrude edges to faces and edges +const edge_to_extruded_edge=[6,8,10,12] +const edge_to_extruded_face=[1,2,3,4] +# 3. Extrude interior to upper face +const cell_to_extruded_cell=[1] +const cell_to_extruded_face=[6] + +function get_bottom_to_extruded_face_lids(Db,De) + if (Db==0 && De==0) + corner_to_extruded_corner + elseif (Db==0 && De==1) + corner_to_extruded_edge + elseif (Db==1 && De==1) + edge_to_extruded_edge + elseif (Db==1 && De==2) + edge_to_extruded_face + elseif (Db==2 && De==2) + cell_to_extruded_face + elseif (Db==2 && De==3) + cell_to_extruded_cell + else + @assert false + end +end + +const corner_to_corner = [1,3,5,7] +const edge_to_edge = [5,7,9,11] +const cell_to_face = [5] +function get_bottom_to_overlapping_face_lids(D) + if (D==0) + corner_to_corner + elseif (D==1) + edge_to_edge + elseif (D==2) + cell_to_face + else + @assert false + end +end + +function transfer_entity_ids!(target_entity_ids, + scell_faces, + tcell_faces, + source_to_target_lid, + source_entity_ids, + offset) + for (blid,bgid) in enumerate(scell_faces) + sentity_id=source_entity_ids[bgid] + egid=tcell_faces[source_to_target_lid[blid]] + target_entity_ids[egid]=sentity_id+offset + end +end + +function generate_face_labeling(pXest_type::P6estType, + parts, + cell_prange, + coarse_discrete_model::DiscreteModel{2,2}, + topology, + ptr_pXest, + ptr_pXest_ghost) + + Dc=3 + + pXest = ptr_pXest[] + pXest_ghost = ptr_pXest_ghost[] + + p4est_type=P4estType() + ptr_p4est_connectivity = pXest.connectivity[].conn4 + ptr_p4est = pXest.columns + ptr_p4est_ghost = setup_pXest_ghost(p4est_type,ptr_p4est) + ptr_p4est_lnodes = setup_pXest_lnodes_nonconforming(p4est_type, ptr_p4est, ptr_p4est_ghost) + + bottom_boundary_model,_=setup_non_conforming_distributed_discrete_model(p4est_type, + parts, + coarse_discrete_model, + ptr_p4est_connectivity, + ptr_p4est, + ptr_p4est_ghost, + ptr_p4est_lnodes) + + pXest_ghost_destroy(p4est_type, ptr_p4est_ghost) + pXest_lnodes_destroy(p4est_type, ptr_p4est_lnodes) + + bottom_boundary_model_topology = map(local_views(bottom_boundary_model)) do model + Gridap.Geometry.get_grid_topology(model) + end + + bottom_boundary_model_labeling = map(local_views(bottom_boundary_model)) do model + Gridap.Geometry.get_face_labeling(model) + end + + coarse_face_labeling = get_face_labeling(coarse_discrete_model) + max_entity_id = _compute_max_entity_id(coarse_face_labeling) + + offset_intermediate_entities = max_entity_id + offset_top_entities = 2*max_entity_id + + faces_to_entity=map(topology, + bottom_boundary_model_topology, + bottom_boundary_model_labeling) do topology, btopology, blabeling + + bcell_faces = [Gridap.Geometry.get_faces(btopology,2,i) for i in 0:1] + bcell_caches = [array_cache(bcell_faces[i]) for i=1:length(bcell_faces)] + cell_faces = [Gridap.Geometry.get_faces(topology,Dc,i) for i in 0:2] + cell_caches = [array_cache(cell_faces[i]) for i=1:length(cell_faces)] + bface_entity = [Gridap.Geometry.get_face_entity(blabeling,i) for i=0:2] + + num_vertices=Gridap.Geometry.num_faces(topology,0) + vertex_to_entity=zeros(Int,num_vertices) + + num_edgets=Gridap.Geometry.num_faces(topology,1) + edget_to_entity=zeros(Int,num_edgets) + + num_faces=Gridap.Geometry.num_faces(topology,Dc-1) + facet_to_entity=zeros(Int,num_faces) + + num_cells=Gridap.Geometry.num_faces(topology,Dc) + cell_to_entity=zeros(Int,num_cells) + + eface_entity = [vertex_to_entity,edget_to_entity,facet_to_entity,cell_to_entity] + + gcell=1 + qcell=1 + + num_trees = Cint(pXest.columns[].connectivity[].num_trees) + + # Go over trees + for itree = 0:num_trees-1 + tree = pXest_tree_array_index(pXest_type,pXest,itree)[] + num_quads = Cint(tree.quadrants.elem_count) + + # Go over columns of current tree + for iquad=0:num_quads-1 + + current_bcell_faces = [getindex!(bcell_caches[i],bcell_faces[i],qcell) for i=1:length(bcell_caches)] + current_cell_faces = [getindex!(cell_caches[i],cell_faces[i],gcell) for i=1:length(cell_caches)] + + # Transfer entity IDs from the bottom 2D cell to the + # bottom of first 3D cell in the column + for d=0:1 + b2oflids=get_bottom_to_overlapping_face_lids(d) + transfer_entity_ids!(eface_entity[d+1], + current_bcell_faces[d+1], + current_cell_faces[d+1], + b2oflids, + bface_entity[d+1], + 0) + end + b2oflids=get_bottom_to_overlapping_face_lids(2) + transfer_entity_ids!(eface_entity[3], + [qcell], + current_cell_faces[3], + b2oflids, + bface_entity[2], + 0) + + q = pXest_quadrant_array_index(pXest_type,tree,iquad) + f,l=P6EST_COLUMN_GET_RANGE(q[]) + + # Loop over layers within current column + for layer=f:l-1 + current_cell_faces = [getindex!(cell_caches[i],cell_faces[i],gcell) for i=1:length(cell_caches)] + + + # Transfer entity IDs from the bottom of 3D cell to the rest of the cell by extrusion + for db=0:2 + for de=db:db+1 + b2oflids=get_bottom_to_extruded_face_lids(db,de) + if (layer==l-1 && db==de ) + offset=offset_top_entities + else + offset=offset_intermediate_entities + end + if (db==2 && de==3) + transfer_entity_ids!(eface_entity[de+1], + [qcell], + [gcell], + b2oflids, + bface_entity[db+1], + offset) + elseif (db==2) + transfer_entity_ids!(eface_entity[de+1], + [qcell], + current_cell_faces[de+1], + b2oflids, + bface_entity[db+1], + offset) + else + @debug "[$(MPI.Comm_rank(MPI.COMM_WORLD))] qcell=$(qcell) gcell=$(gcell) de=$(de) db=$(db) current_bcell_faces[db+1]=$(current_bcell_faces[db+1]) current_cell_faces[de+1]=$(current_cell_faces[de+1]) b2oflids=$(b2oflids) bface_entity[db+1]=$(bface_entity[db+1]) eface_entity[de+1]=$(eface_entity[de+1])" + transfer_entity_ids!(eface_entity[de+1], + current_bcell_faces[db+1], + current_cell_faces[de+1], + b2oflids, + bface_entity[db+1], + offset) + @debug "[$(MPI.Comm_rank(MPI.COMM_WORLD))] eface_entity[de+1]=$(eface_entity[de+1])" + end + end + end + gcell=gcell+1 + end + qcell=qcell+1 + end + end + vertex_to_entity, edget_to_entity, facet_to_entity, cell_to_entity + end + + vertex_to_entity = map(x->x[1] , faces_to_entity) + edget_to_entity = map(x->x[2] , faces_to_entity) + facet_to_entity = map(x->x[Dc] , faces_to_entity) + cell_to_entity = map(x->x[Dc+1], faces_to_entity) + + function cell_to_faces(topology,cell_dim,face_dim) + map(topology) do topology + Gridap.Geometry.get_faces(topology,cell_dim,face_dim) + end + end + +faces_to_entity=[vertex_to_entity,edget_to_entity,facet_to_entity,cell_to_entity] + +face_labeling = + map(bottom_boundary_model_labeling,faces_to_entity...) do blabeling, faces_to_entity... + d_to_dface_to_entity = Vector{Vector{Int}}(undef,Dc+1) + d_to_dface_to_entity[1] = faces_to_entity[1] + d_to_dface_to_entity[2] = faces_to_entity[2] + + d_to_dface_to_entity[Dc] = faces_to_entity[Dc] + d_to_dface_to_entity[Dc+1] = faces_to_entity[Dc+1] + + bottom_interior_tag=findfirst(x->x=="interior",coarse_discrete_model.face_labeling.tag_to_name) + @assert bottom_interior_tag != nothing + bottom_interior_entities = coarse_discrete_model.face_labeling.tag_to_entities[bottom_interior_tag] + + intermediate_interior_entities = [e+offset_intermediate_entities for e in bottom_interior_entities] + + boundary_intermediate_entities_set=setdiff( + Set(collect(Int32(offset_intermediate_entities+1):Int32(offset_top_entities))), + Set(intermediate_interior_entities)) + + boundary_intermediate_entities=[e for e in boundary_intermediate_entities_set] + + intermediate_interior_entities_set=setdiff(Set(collect(offset_intermediate_entities+1:offset_top_entities)), + boundary_intermediate_entities_set) + + # Tags: bottom-boundary (1), intermediate boundary (2), top boundary (3), interior (4) + tag_to_entities = Vector{Vector{Int32}}(undef,4) + tag_to_entities[1] = collect(1:max_entity_id) + + # All the intermediate entities except those which have been extruded from the interior of the bottom model + tag_to_entities[2] = boundary_intermediate_entities + + tag_to_entities[3] = collect(offset_top_entities+1:3*max_entity_id) + + # All the intermediate entities except boundary_intermediate_entities + tag_to_entities[4] = [e for e in intermediate_interior_entities_set] + tag_to_name = ["bottom_boundary", "intermediate_boundary", "top_boundary", "interior"] + + face_labeling=Gridap.Geometry.FaceLabeling(d_to_dface_to_entity, + tag_to_entities, + tag_to_name) + end + face_labeling +end diff --git a/src/GridapP4est.jl b/src/GridapP4est.jl index e1d41c2..be542e7 100644 --- a/src/GridapP4est.jl +++ b/src/GridapP4est.jl @@ -13,14 +13,17 @@ module GridapP4est const PArrays = PartitionedArrays include("Environment.jl") + include("PXestTypeMethods.jl") include("UniformlyRefinedForestOfOctreesDiscreteModels.jl") include("OctreeDistributedDiscreteModels.jl") + include("AnisotropicallyAdapted3DDistributedDiscreteModels.jl") include("GridapFixes.jl") include("FESpaces.jl") include("AdaptivityFlagsMarkingStrategies.jl") export UniformlyRefinedForestOfOctreesDiscreteModel export OctreeDistributedDiscreteModel + export AnisotropicallyAdapted3DDistributedDiscreteModel export adapt export refine export coarsen diff --git a/src/OctreeDistributedDiscreteModels.jl b/src/OctreeDistributedDiscreteModels.jl index 11a62b6..1a0012d 100644 --- a/src/OctreeDistributedDiscreteModels.jl +++ b/src/OctreeDistributedDiscreteModels.jl @@ -40,6 +40,7 @@ mutable struct OctreeDistributedDiscreteModel{Dc,Dp,A,B,C,D,E,F} <: GridapDistri coarse_model :: D ptr_pXest_connectivity :: E ptr_pXest :: F + pXest_type :: PXestType # The model for which this variable is true, is the one # ultimately responsible for deallocating the pXest_connectivity @@ -58,6 +59,7 @@ mutable struct OctreeDistributedDiscreteModel{Dc,Dp,A,B,C,D,E,F} <: GridapDistri coarse_model, ptr_pXest_connectivity, ptr_pXest, + pXest_type::PXestType, owns_ptr_pXest_connectivity::Bool, gc_ref) @@ -78,6 +80,7 @@ mutable struct OctreeDistributedDiscreteModel{Dc,Dp,A,B,C,D,E,F} <: GridapDistri coarse_model, ptr_pXest_connectivity, ptr_pXest, + pXest_type, owns_ptr_pXest_connectivity, gc_ref) Init(model) @@ -92,6 +95,7 @@ function OctreeDistributedDiscreteModel( coarse_model, ptr_pXest_connectivity, ptr_pXest, + pXest_type, owns_ptr_pXest_connectivity, gc_ref) where {Dc,Dp} @@ -103,16 +107,24 @@ function OctreeDistributedDiscreteModel( coarse_model, ptr_pXest_connectivity, ptr_pXest, + pXest_type, owns_ptr_pXest_connectivity, gc_ref) end +function _dim_to_pXest_type(Dc) + Dc==2 ? P4estType() : P8estType() +end + function OctreeDistributedDiscreteModel(parts::AbstractVector{<:Integer}, coarse_model::DiscreteModel{Dc,Dp}, num_uniform_refinements) where {Dc,Dp} comm = parts.comm if GridapDistributed.i_am_in(comm) + + pXest_type = _dim_to_pXest_type(Dc) + ptr_pXest_connectivity, ptr_pXest, ptr_pXest_ghost, @@ -127,8 +139,8 @@ function OctreeDistributedDiscreteModel(parts::AbstractVector{<:Integer}, ptr_pXest, ptr_pXest_ghost, ptr_pXest_lnodes) - pXest_lnodes_destroy(Val{Dc},ptr_pXest_lnodes) - pXest_ghost_destroy(Val{Dc},ptr_pXest_ghost) + pXest_lnodes_destroy(pXest_type,ptr_pXest_lnodes) + pXest_ghost_destroy(pXest_type,ptr_pXest_ghost) non_conforming_glue = _create_conforming_model_non_conforming_glue(dmodel) @@ -140,6 +152,7 @@ function OctreeDistributedDiscreteModel(parts::AbstractVector{<:Integer}, coarse_model, ptr_pXest_connectivity, ptr_pXest, + pXest_type, true, nothing) else @@ -170,6 +183,7 @@ function VoidOctreeDistributedDiscreteModel(coarse_model::DiscreteModel{Dc,Dp},p coarse_model, ptr_pXest_connectivity, nothing, + _dim_to_pXest_type(Dc), true, nothing) end @@ -183,6 +197,7 @@ function VoidOctreeDistributedDiscreteModel(model::OctreeDistributedDiscreteMode model.coarse_model, model.ptr_pXest_connectivity, nothing, + _dim_to_pXest_type(Dc), false, model) end @@ -198,17 +213,17 @@ GridapDistributed.get_face_gids(model::OctreeDistributedDiscreteModel,dim::Integ function octree_distributed_discrete_model_free!(model::VoidOctreeDistributedDiscreteModel{Dc}) where Dc if (model.owns_ptr_pXest_connectivity) - pXest_connectivity_destroy(Val{Dc},model.ptr_pXest_connectivity) + pXest_connectivity_destroy(model.pXest_type,model.ptr_pXest_connectivity) end return nothing end function octree_distributed_discrete_model_free!(model::OctreeDistributedDiscreteModel{Dc}) where Dc if !isa(model.ptr_pXest,Nothing) - pXest_destroy(Val{Dc},model.ptr_pXest) + pXest_destroy(model.pXest_type,model.ptr_pXest) end if (model.owns_ptr_pXest_connectivity) - pXest_connectivity_destroy(Val{Dc},model.ptr_pXest_connectivity) + pXest_connectivity_destroy(model.pXest_type,model.ptr_pXest_connectivity) end return nothing end @@ -225,43 +240,6 @@ end ################################################################### # Private methods -function pXest_copy(::Type{Val{Dc}}, ptr_pXest) where Dc - if (Dc==2) - p4est_copy(ptr_pXest, Cint(0)) - else - p8est_copy(ptr_pXest, Cint(0)) - end -end - -function pXest_partition!(::Type{Val{Dc}}, ptr_pXest) where Dc - if (Dc==2) - # The 1 here is required to avoid that the children of the - # same parent are assigned to different partitions - p4est_partition(ptr_pXest, 0, C_NULL) - else - p8est_partition(ptr_pXest, 0, C_NULL) - end -end - -function pXest_balance!(::Type{Val{Dc}}, ptr_pXest; k_2_1_balance=0) where Dc - if (Dc==2) - if (k_2_1_balance==0) - p4est_balance(ptr_pXest, P4est_wrapper.P4EST_CONNECT_FULL, C_NULL) - else - p4est_balance(ptr_pXest, P4est_wrapper.P4EST_CONNECT_FACE, C_NULL) - end - else - if (k_2_1_balance==0) - p8est_balance(ptr_pXest, P4est_wrapper.P8EST_CONNECT_FULL, C_NULL) - elseif (k_2_1_balance==1) - p8est_balance(ptr_pXest, P4est_wrapper.P8EST_CONNECT_EDGE, C_NULL) - else - @assert k_2_1_balance==2 - p8est_balance(ptr_pXest, P4est_wrapper.P8EST_CONNECT_FACE, C_NULL) - end - end -end - function pXest_partition_given!(::Type{Val{Dc}}, ptr_pXest, new_num_cells_per_part) where Dc if (Dc==2) p4est_partition_given(ptr_pXest, new_num_cells_per_part) @@ -289,14 +267,6 @@ function pXest_uniformly_refine!(::Type{Val{Dc}}, ptr_pXest) where Dc end end -function pXest_refine!(::Type{Val{Dc}}, ptr_pXest, refine_fn_c, refine_replace_fn_c; init_fn_c=C_NULL) where Dc - if (Dc==2) - p4est_refine_ext(ptr_pXest, Cint(0), Cint(-1), refine_fn_c, init_fn_c, refine_replace_fn_c) - else - p8est_refine_ext(ptr_pXest, Cint(0), Cint(-1), refine_fn_c, init_fn_c, refine_replace_fn_c) - end -end - function pXest_uniformly_coarsen!(::Type{Val{Dc}}, ptr_pXest) where Dc # Coarsen callbacks function coarsen_fn_2d(::Ptr{p4est_t},::p4est_topidx_t,::Ptr{Ptr{p4est_quadrant_t}}) @@ -315,28 +285,47 @@ function pXest_uniformly_coarsen!(::Type{Val{Dc}}, ptr_pXest) where Dc end end -function get_num_children(::Type{Val{Dc}}) where Dc - 2^Dc +function get_num_children(::P4estType,::PXestUniformRefinementRuleType) + 4 end -function pXest_coarsen!(::Type{Val{Dc}}, ptr_pXest, coarsen_fn_c) where Dc - if (Dc==2) - p4est_coarsen(ptr_pXest, Cint(0), coarsen_fn_c, C_NULL) - else - p8est_coarsen(ptr_pXest, Cint(0), coarsen_fn_c, C_NULL) - end +function get_num_children(::P8estType,::PXestUniformRefinementRuleType) + 8 end +function get_num_children(::P6estType,::PXestVerticalRefinementRuleType) + 2 +end -function pXest_reset_data!(::Type{Val{Dc}}, ptr_pXest, data_size, init_fn_c, user_pointer) where Dc - if (Dc==2) - p4est_reset_data(ptr_pXest, data_size, init_fn_c, user_pointer) - else - p8est_reset_data(ptr_pXest, data_size, init_fn_c, user_pointer) - end +function get_num_children(::P6estType,::PXestHorizontalRefinementRuleType) + 4 end -function pXest_update_flags!(::Type{Val{Dc}}, ptr_pXest_old, ptr_pXest_new) where Dc +function get_num_children(::Type{Val{Dc}}) where Dc + 2^Dc +end + +function get_refinement_rule(::P4estType,::PXestUniformRefinementRuleType) + reffe = LagrangianRefFE(Float64,QUAD,1) + Gridap.Adaptivity.RefinementRule(reffe,2) +end + +function get_refinement_rule(::P8estType,::PXestUniformRefinementRuleType) + reffe = LagrangianRefFE(Float64,HEX,1) + Gridap.Adaptivity.RefinementRule(reffe,2) +end + +function get_refinement_rule(::P6estType,::PXestVerticalRefinementRuleType) + reffe = LagrangianRefFE(Float64,HEX,1) + Gridap.Adaptivity.RefinementRule(reffe,(2,1,1)) +end + +function get_refinement_rule(::P6estType,::PXestHorizontalRefinementRuleType) + reffe = LagrangianRefFE(Float64,HEX,1) + Gridap.Adaptivity.RefinementRule(reffe,(1,2,2)) +end + +function pXest_update_flags!(pXest_type::P4P8estType, ptr_pXest_old, ptr_pXest_new) pXest_old = ptr_pXest_old[] pXest_new = ptr_pXest_new[] flags=unsafe_wrap(Array, @@ -346,36 +335,34 @@ function pXest_update_flags!(::Type{Val{Dc}}, ptr_pXest_old, ptr_pXest_new) wher num_trees = Cint(pXest_old.connectivity[].num_trees) @assert num_trees == Cint(pXest_new.connectivity[].num_trees) - num_children = get_num_children(Val{Dc}) - global_iquad_new = 0 + Dc=num_cell_dims(pXest_type) + + num_children = get_num_children(pXest_type, PXestUniformRefinementRuleType()) global_iquad_old = 0 for itree = 0:num_trees-1 - tree_old = _pXest_tree_array_index(Val{Dc},pXest_old.trees,itree)[] - tree_new = _pXest_tree_array_index(Val{Dc},pXest_new.trees,itree)[] + tree_old = pXest_tree_array_index(pXest_type,pXest_old,itree)[] + tree_new = pXest_tree_array_index(pXest_type,pXest_new,itree)[] num_quads_old = Cint(tree_old.quadrants.elem_count) local_iquad_old=0 local_iquad_new=0 while local_iquad_old < num_quads_old - q_old = _pXest_quadrant_array_index(Val{Dc},tree_old.quadrants,local_iquad_old) - q_new = _pXest_quadrant_array_index(Val{Dc},tree_new.quadrants,local_iquad_new) - if (_pXest_quadrant_compare(Val{Dc},q_old,q_new) == 0) # q_old was not refined nor coarsened + q_old = pXest_quadrant_array_index(pXest_type,tree_old,local_iquad_old) + q_new = pXest_quadrant_array_index(pXest_type,tree_new,local_iquad_new) + if (pXest_quadrant_is_equal(pXest_type,q_old,q_new)) # q_old was not refined nor coarsened flags[global_iquad_old+1] = nothing_flag - global_iquad_new += 1 global_iquad_old += 1 local_iquad_new += 1 local_iquad_old += 1 - elseif (_pXest_quadrant_is_parent(Val{Dc},q_old,q_new)!=0) # q_old was refined + elseif (pXest_quadrant_is_parent(pXest_type,q_old,q_new)) # q_old was refined flags[global_iquad_old+1] = refine_flag - global_iquad_new += num_children global_iquad_old += 1 local_iquad_new += num_children local_iquad_old += 1 - elseif (_pXest_quadrant_is_parent(Val{Dc},q_new,q_old)!=0) # q_old and its siblings were coarsened + elseif (pXest_quadrant_is_parent(pXest_type,q_new,q_old)) # q_old and its siblings were coarsened for i=0:num_children-1 flags[global_iquad_old+i+1] = coarsen_flag end global_iquad_old += num_children - global_iquad_new += 1 local_iquad_old += num_children local_iquad_new += 1 else @@ -385,6 +372,65 @@ function pXest_update_flags!(::Type{Val{Dc}}, ptr_pXest_old, ptr_pXest_new) wher end end +function p6est_vertically_adapt_update_flags!(ptr_pXest_old, ptr_pXest_new) + pXest_old = ptr_pXest_old[] + pXest_new = ptr_pXest_new[] + flags=unsafe_wrap(Array, + Ptr{Cint}(pXest_old.user_pointer), + pXest_old.layers[].elem_count) + + num_trees = Cint(pXest_old.columns[].connectivity[].num_trees) + @assert num_trees == Cint(pXest_new.columns[].connectivity[].num_trees) + + pXest_type=P6estType() + num_children = get_num_children(pXest_type, PXestVerticalRefinementRuleType()) + global_icell_old = 0 + for itree = 0:num_trees-1 + tree_old = pXest_tree_array_index(pXest_type,pXest_old,itree)[] + tree_new = pXest_tree_array_index(pXest_type,pXest_new,itree)[] + num_quads = Cint(tree_old.quadrants.elem_count) + num_quads_new = Cint(tree_new.quadrants.elem_count) + @assert num_quads == num_quads_new + + local_iquad=0 + # Loop over quadrants + for local_iquad=0:num_quads-1 + q_old = pXest_quadrant_array_index(pXest_type,tree_old,local_iquad) + q_new = pXest_quadrant_array_index(pXest_type,tree_new,local_iquad) + + f_old,l_old=P6EST_COLUMN_GET_RANGE(q_old[]) + f_new,l_new=P6EST_COLUMN_GET_RANGE(q_new[]) + i_old=f_old + i_new=f_new + # Loop over layers within current column + while i_old tuple_of_arrays @@ -970,9 +1026,7 @@ function _compute_fine_to_coarse_model_glue( ref_grid = UnstructuredGrid(compute_reference_grid(polytope,partition)) rrule_nothing_flag = Gridap.Adaptivity.RefinementRule(Gridap.Adaptivity.WithoutRefinement(),polytope,ref_grid) - - reffe = LagrangianRefFE(Float64,polytope,1) - rrule_refinement_flag = Gridap.Adaptivity.RefinementRule(reffe,2) + rrule_refinement_flag = get_refinement_rule(pXest_type,pXest_refinement_rule_type) coarse_cell_to_rrule = map(x -> (x==nothing_flag) ? 1 : 2,flags) rrules = Gridap.Arrays.CompressedArray([rrule_nothing_flag,rrule_refinement_flag],coarse_cell_to_rrule) @debug "[$(MPI.Comm_rank(MPI.COMM_WORLD))] fine_to_coarse_faces_map[end]: $(fine_to_coarse_faces_map[end])" @@ -984,15 +1038,17 @@ function _compute_fine_to_coarse_model_glue( return glue end -function _process_owned_cells_fine_to_coarse_model_glue(cmodel::DiscreteModel{Dc}, +function _process_owned_cells_fine_to_coarse_model_glue(pXest_type, + pXest_refinement_rule_type, + cmodel::DiscreteModel{Dc}, fmodel::DiscreteModel{Dc}, cpartition, fpartition, flags, coarsen) where Dc - function _setup_fine_to_coarse_faces_map_table(Dc,flags,num_o_c_cells,num_f_cells) - num_children = get_num_children(Val{Dc}) + function _setup_fine_to_coarse_faces_map_table(pXest_type,pXest_refinement_rule_type,flags,num_o_c_cells,num_f_cells) + num_children = get_num_children(pXest_type, pXest_refinement_rule_type) # Count cell children: fine_to_coarse_faces_map_ptrs = Vector{Int}(undef,num_f_cells+1) @@ -1076,9 +1132,9 @@ function _process_owned_cells_fine_to_coarse_model_glue(cmodel::DiscreteModel{Dc Gridap.Arrays.Table(fcell_to_child_id_data, fine_to_coarse_faces_map_ptrs) end - function _setup_fine_to_coarse_faces_map_vector!(fine_to_coarse_faces_map,fcell_to_child_id,Dc,flags,num_o_c_cells) + function _setup_fine_to_coarse_faces_map_vector!(pXest_type,pXest_refinement_rule_type,fine_to_coarse_faces_map,fcell_to_child_id,flags,num_o_c_cells) # Go over all cells of coarse grid portion - num_children = get_num_children(Val{Dc}) + num_children = get_num_children(pXest_type,pXest_refinement_rule_type) c = 1 for cell = 1:num_o_c_cells if flags[cell]==refine_flag @@ -1103,7 +1159,7 @@ function _process_owned_cells_fine_to_coarse_model_glue(cmodel::DiscreteModel{Dc ftopology = Gridap.Geometry.get_grid_topology(fmodel) if coarsen fine_to_coarse_faces_map = Vector{Gridap.Arrays.Table{Int,Vector{Int},Vector{Int}}}(undef,Dc+1) - a,b = _setup_fine_to_coarse_faces_map_table(Dc,flags,num_o_c_cells,num_f_cells) + a,b = _setup_fine_to_coarse_faces_map_table(pXest_type,pXest_refinement_rule_type,flags,num_o_c_cells,num_f_cells) fine_to_coarse_faces_map[Dc+1] = a fcell_to_child_id = b # In the future we should also have here the code to also setup @@ -1112,7 +1168,7 @@ function _process_owned_cells_fine_to_coarse_model_glue(cmodel::DiscreteModel{Dc fine_to_coarse_faces_map = Vector{Vector{Int}}(undef,Dc+1) fine_to_coarse_faces_map[Dc+1] = Vector{Int}(undef,num_f_cells) fcell_to_child_id = Vector{Int}(undef,num_f_cells) - _setup_fine_to_coarse_faces_map_vector!(fine_to_coarse_faces_map[Dc+1],fcell_to_child_id,Dc,flags,num_o_c_cells) + _setup_fine_to_coarse_faces_map_vector!(pXest_type,pXest_refinement_rule_type,fine_to_coarse_faces_map[Dc+1],fcell_to_child_id,flags,num_o_c_cells) for d=1:Dc fine_to_coarse_faces_map[d] = Vector{Int}(undef,Gridap.Geometry.num_faces(ftopology,d-1)) end @@ -1168,7 +1224,7 @@ function Gridap.Adaptivity.refine(model::OctreeDistributedDiscreteModel{Dc,Dp}; old_comm = model.parts.comm if (GridapDistributed.i_am_in(old_comm)) # Copy and refine input p4est - ptr_new_pXest = pXest_copy(Val{Dc}, model.ptr_pXest) + ptr_new_pXest = pXest_copy(model.pXest_type, model.ptr_pXest) pXest_uniformly_refine!(Val{Dc}, ptr_new_pXest) else ptr_new_pXest = nothing @@ -1176,6 +1232,8 @@ function Gridap.Adaptivity.refine(model::OctreeDistributedDiscreteModel{Dc,Dp}; new_comm = isa(parts,Nothing) ? old_comm : parts.comm if GridapDistributed.i_am_in(new_comm) + pXest_type = _dim_to_pXest_type(Dc) + if !isa(parts,Nothing) aux = ptr_new_pXest ptr_new_pXest = _pXest_to_new_comm(Val{Dc}, @@ -1184,7 +1242,7 @@ function Gridap.Adaptivity.refine(model::OctreeDistributedDiscreteModel{Dc,Dp}; model.parts.comm, parts.comm) if GridapDistributed.i_am_in(old_comm) - pXest_destroy(Val{Dc},aux) + pXest_destroy(pXest_type,aux) end end @@ -1202,8 +1260,8 @@ function Gridap.Adaptivity.refine(model::OctreeDistributedDiscreteModel{Dc,Dp}; ptr_pXest_ghost, ptr_pXest_lnodes) - pXest_lnodes_destroy(Val{Dc},ptr_pXest_lnodes) - pXest_ghost_destroy(Val{Dc},ptr_pXest_ghost) + pXest_lnodes_destroy(pXest_type,ptr_pXest_lnodes) + pXest_ghost_destroy(pXest_type,ptr_pXest_ghost) dglue = _compute_fine_to_coarse_model_glue(model.parts, model.dmodel, @@ -1219,6 +1277,7 @@ function Gridap.Adaptivity.refine(model::OctreeDistributedDiscreteModel{Dc,Dp}; model.coarse_model, model.ptr_pXest_connectivity, ptr_new_pXest, + pXest_type, false, model) @@ -1234,200 +1293,28 @@ end function _refine_coarsen_balance!(model::OctreeDistributedDiscreteModel{Dc,Dp}, refinement_and_coarsening_flags::MPIArray{<:Vector}) where {Dc,Dp} - # Variables which are updated accross calls to init_fn_callback_2d - current_quadrant_index_within_tree = Cint(0) - current_quadrant_index_among_trees = Cint(0) - - # This C callback function is called once per quadtree quadrant. Here we are assuming - # that p4est->user_pointer has been set prior to the first call to this call - # back function to an array of ints with as many entries as forest quadrants. This call back function - # initializes the quadrant->p.user_data void * pointer of all quadrants such that it - # points to the corresponding entry in the global array mentioned in the previous sentence. - if (Dc==2) - function init_fn_callback_2d(forest_ptr::Ptr{p4est_t}, - which_tree::p4est_topidx_t, - quadrant_ptr::Ptr{p4est_quadrant_t}) - # Extract a reference to the tree which_tree - forest = forest_ptr[] - tree = p4est_tree_array_index(forest.trees, which_tree)[] - quadrant = quadrant_ptr[] - q = P4est_wrapper.p4est_quadrant_array_index(tree.quadrants, current_quadrant_index_within_tree) - @assert p4est_quadrant_compare(q, quadrant_ptr) == 0 - user_data = unsafe_wrap(Array, - Ptr{Cint}(forest.user_pointer), - current_quadrant_index_among_trees+1)[current_quadrant_index_among_trees+1] - unsafe_store!(Ptr{Cint}(quadrant.p.user_data), user_data, 1) - current_quadrant_index_within_tree = (current_quadrant_index_within_tree + 1) % (tree.quadrants.elem_count) - current_quadrant_index_among_trees = current_quadrant_index_among_trees+1 - return nothing - end - init_fn_callback_2d_c = @cfunction($init_fn_callback_2d, - Cvoid, (Ptr{p4est_t}, p4est_topidx_t, Ptr{p4est_quadrant_t})) - init_fn_callback_c = init_fn_callback_2d_c - - - function coarsen_callback_2d(forest_ptr::Ptr{p4est_t}, - which_tree::p4est_topidx_t, - quadrant_ptr::Ptr{Ptr{p4est_quadrant_t}}) - - num_children=get_num_children(Val{2}) - quadrants=unsafe_wrap(Array, quadrant_ptr, num_children) - coarsen=Cint(1) - for quadrant_index=1:num_children - quadrant = quadrants[quadrant_index][] - # I have noticed that new quadrants created as by-product - # of the refininement process have quadrant.p.user_data == C_NULL - # Not sure why ... The following if-end takes care of this. - if (quadrant.p.user_data) == C_NULL - return Cint(0) - end - is_coarsen_flag=(unsafe_wrap(Array,Ptr{Cint}(quadrant.p.user_data),1)[])==coarsen_flag - if (!is_coarsen_flag) - return Cint(0) - end - end - return coarsen - end - coarsen_fn_callback_2d_c = @cfunction($coarsen_callback_2d, - Cint, (Ptr{p4est_t}, p4est_topidx_t, Ptr{Ptr{p4est_quadrant_t}})) - coarsen_fn_callback_c = coarsen_fn_callback_2d_c - - - function refine_replace_callback_2d(::Ptr{p4est_t}, - which_tree::p4est_topidx_t, - num_outgoing::Cint, - outgoing_ptr::Ptr{Ptr{p4est_quadrant_t}}, - num_incoming::Cint, - incoming_ptr::Ptr{Ptr{p4est_quadrant_t}}) - num_children=get_num_children(Val{2}) - @assert num_outgoing==1 - @assert num_incoming==num_children - outgoing=unsafe_wrap(Array, outgoing_ptr, 1) - quadrant = outgoing[1][] - incoming=unsafe_wrap(Array, incoming_ptr, num_children) - for quadrant_index=1:num_children - quadrant = incoming[quadrant_index][] - if (quadrant.p.user_data) != C_NULL - unsafe_store!(Ptr{Cint}(quadrant.p.user_data), nothing_flag, 1) - end - end - end - - refine_replace_callback_2d_c = - @cfunction($refine_replace_callback_2d, Cvoid, (Ptr{p4est_t}, - p4est_topidx_t, - Cint, - Ptr{Ptr{p4est_quadrant_t}}, - Cint, - Ptr{Ptr{p4est_quadrant_t}})) - refine_replace_callback_c = refine_replace_callback_2d_c - - else - @assert Dc==3 - function init_fn_callback_3d(forest_ptr::Ptr{p8est_t}, - which_tree::p4est_topidx_t, - quadrant_ptr::Ptr{p8est_quadrant_t}) - # Extract a reference to the tree which_tree - forest = forest_ptr[] - tree = p8est_tree_array_index(forest.trees, which_tree)[] - quadrant = quadrant_ptr[] - q = P4est_wrapper.p8est_quadrant_array_index(tree.quadrants, current_quadrant_index_within_tree) - @assert p8est_quadrant_compare(q, quadrant_ptr) == 0 - user_data = unsafe_wrap(Array, - Ptr{Cint}(forest.user_pointer), - current_quadrant_index_among_trees+1)[current_quadrant_index_among_trees+1] - unsafe_store!(Ptr{Cint}(quadrant.p.user_data), user_data, 1) - current_quadrant_index_within_tree = (current_quadrant_index_within_tree + 1) % (tree.quadrants.elem_count) - current_quadrant_index_among_trees = current_quadrant_index_among_trees+1 - return nothing - end - init_fn_callback_3d_c = @cfunction($init_fn_callback_3d, - Cvoid, (Ptr{p8est_t}, p4est_topidx_t, Ptr{p8est_quadrant_t})) - init_fn_callback_c = init_fn_callback_3d_c - - - function coarsen_callback_3d(forest_ptr::Ptr{p8est_t}, - which_tree::p4est_topidx_t, - quadrant_ptr::Ptr{Ptr{p8est_quadrant_t}}) - - num_children=get_num_children(Val{3}) - quadrants=unsafe_wrap(Array, quadrant_ptr, num_children) - coarsen=Cint(1) - for quadrant_index=1:num_children - quadrant = quadrants[quadrant_index][] - # I have noticed that new quadrants created as by-product - # of the refininement process have quadrant.p.user_data == C_NULL - # Not sure why ... The following if-end takes care of this. - if (quadrant.p.user_data) == C_NULL - return Cint(0) - end - is_coarsen_flag=(unsafe_wrap(Array,Ptr{Cint}(quadrant.p.user_data),1)[])==coarsen_flag - if (!is_coarsen_flag) - return Cint(0) - end - end - return coarsen - end - coarsen_fn_callback_3d_c = @cfunction($coarsen_callback_3d, - Cint, (Ptr{p8est_t}, p4est_topidx_t, Ptr{Ptr{p8est_quadrant_t}})) - coarsen_fn_callback_c = coarsen_fn_callback_3d_c - - function refine_replace_callback_3d(::Ptr{p8est_t}, - which_tree::p4est_topidx_t, - num_outgoing::Cint, - outgoing_ptr::Ptr{Ptr{p8est_quadrant_t}}, - num_incoming::Cint, - incoming_ptr::Ptr{Ptr{p8est_quadrant_t}}) - num_children=get_num_children(Val{3}) - @assert num_outgoing==1 - @assert num_incoming==num_children - outgoing=unsafe_wrap(Array, outgoing_ptr, 1) - quadrant = outgoing[1][] - incoming=unsafe_wrap(Array, incoming_ptr, num_children) - for quadrant_index=1:num_children - quadrant = incoming[quadrant_index][] - if (quadrant.p.user_data) != C_NULL - unsafe_store!(Ptr{Cint}(quadrant.p.user_data), nothing_flag, 1) - end - end - end - - refine_replace_callback_3d_c = - @cfunction($refine_replace_callback_3d, Cvoid, (Ptr{p8est_t}, - p4est_topidx_t, - Cint, - Ptr{Ptr{p8est_quadrant_t}}, - Cint, - Ptr{Ptr{p8est_quadrant_t}})) - - refine_replace_callback_c = refine_replace_callback_3d_c - end # callback if + pXest_type = model.pXest_type + init_fn_callback_c = pXest_reset_callbacks(pXest_type) + coarsen_fn_callback_c = pXest_coarsen_callbacks(pXest_type) + refine_callback_c,refine_replace_callback_c = pXest_refine_callbacks(pXest_type) map(model.dmodel.models,refinement_and_coarsening_flags) do lmodel, flags # The length of the local flags array has to match the number of # cells in the model. This includes both owned and ghost cells. # Only the flags for owned cells are actually taken into account. @assert num_cells(lmodel)==length(flags) - pXest_reset_data!(Val{Dc}, model.ptr_pXest, Cint(sizeof(Cint)), init_fn_callback_c, pointer(flags)) + pXest_reset_data!(pXest_type, model.ptr_pXest, Cint(sizeof(Cint)), init_fn_callback_c, pointer(flags)) end - - function refine_callback_2d(::Ptr{p4est_t}, - which_tree::p4est_topidx_t, - quadrant_ptr::Ptr{p4est_quadrant_t}) - quadrant = quadrant_ptr[] - return Cint(unsafe_wrap(Array, Ptr{Cint}(quadrant.p.user_data), 1)[] == refine_flag) - end - refine_callback_2d_c = @cfunction($refine_callback_2d, Cint, (Ptr{p4est_t}, p4est_topidx_t, Ptr{p4est_quadrant_t})) # Copy input p4est, refine and balance - ptr_new_pXest = pXest_copy(Val{Dc}, model.ptr_pXest) - pXest_refine!(Val{Dc}, ptr_new_pXest, - refine_callback_2d_c, + ptr_new_pXest = pXest_copy(pXest_type, model.ptr_pXest) + pXest_refine!(pXest_type, ptr_new_pXest, + refine_callback_c, refine_replace_callback_c) - pXest_coarsen!(Val{Dc}, ptr_new_pXest, coarsen_fn_callback_c) - pXest_balance!(Val{Dc}, ptr_new_pXest) - pXest_update_flags!(Val{Dc},model.ptr_pXest,ptr_new_pXest) + pXest_coarsen!(pXest_type, ptr_new_pXest, coarsen_fn_callback_c) + pXest_balance!(pXest_type, ptr_new_pXest) + pXest_update_flags!(pXest_type,model.ptr_pXest,ptr_new_pXest) ptr_new_pXest end @@ -1448,7 +1335,7 @@ function Gridap.Adaptivity.adapt(model::OctreeDistributedDiscreteModel{Dc,Dp}, ptr_pXest_lnodes = setup_pXest_lnodes_nonconforming(Val{Dc}, ptr_new_pXest, ptr_pXest_ghost) # Build fine-grid mesh - fmodel,non_conforming_glue = setup_non_conforming_distributed_discrete_model(Val{Dc}, + fmodel,non_conforming_glue = setup_non_conforming_distributed_discrete_model(model.pXest_type, model.parts, model.coarse_model, model.ptr_pXest_connectivity, @@ -1456,12 +1343,14 @@ function Gridap.Adaptivity.adapt(model::OctreeDistributedDiscreteModel{Dc,Dp}, ptr_pXest_ghost, ptr_pXest_lnodes) - pXest_ghost_destroy(Val{Dc},ptr_pXest_ghost) - pXest_lnodes_destroy(Val{Dc},ptr_pXest_lnodes) - adaptivity_glue = _compute_fine_to_coarse_model_glue(model.parts, - model.dmodel, - fmodel, - _refinement_and_coarsening_flags) + pXest_ghost_destroy(model.pXest_type,ptr_pXest_ghost) + pXest_lnodes_destroy(model.pXest_type,ptr_pXest_lnodes) + adaptivity_glue = _compute_fine_to_coarse_model_glue(model.pXest_type, + PXestUniformRefinementRuleType(), + model.parts, + model.dmodel, + fmodel, + _refinement_and_coarsening_flags) adaptive_models = map(local_views(model), local_views(fmodel), adaptivity_glue) do model, fmodel, glue @@ -1475,6 +1364,7 @@ function Gridap.Adaptivity.adapt(model::OctreeDistributedDiscreteModel{Dc,Dp}, model.coarse_model, model.ptr_pXest_connectivity, ptr_new_pXest, + model.pXest_type, false, model) return ref_model, adaptivity_glue @@ -1483,8 +1373,8 @@ end function Gridap.Adaptivity.coarsen(model::OctreeDistributedDiscreteModel{Dc,Dp}) where {Dc,Dp} comm = model.parts.comm if (GridapDistributed.i_am_in(comm)) - # Copy and refine input p4est - ptr_new_pXest = pXest_copy(Val{Dc}, model.ptr_pXest) + # Copy and coarsen input p4est + ptr_new_pXest = pXest_copy(model.pXest_type, model.ptr_pXest) pXest_uniformly_coarsen!(Val{Dc}, ptr_new_pXest) else ptr_new_pXest=nothing @@ -1504,8 +1394,8 @@ function Gridap.Adaptivity.coarsen(model::OctreeDistributedDiscreteModel{Dc,Dp}) ptr_pXest_ghost, ptr_pXest_lnodes) - pXest_lnodes_destroy(Val{Dc},ptr_pXest_lnodes) - pXest_ghost_destroy(Val{Dc},ptr_pXest_ghost) + pXest_lnodes_destroy(model.pXest_type,ptr_pXest_lnodes) + pXest_ghost_destroy(model.pXest_type,ptr_pXest_ghost) dglue = _compute_fine_to_coarse_model_glue(model.parts, cmodel, @@ -1664,15 +1554,6 @@ function _pXest_to_new_comm_old_supset_new(::Type{Val{Dc}},ptr_pXest, ptr_pXest_ end end - -function _pXest_tree_array_index(::Type{Val{Dc}},trees,itree) where Dc - if (Dc==2) - return p4est_tree_array_index(trees,itree) - elseif (Dc==3) - return p8est_tree_array_index(trees,itree) - end -end - function _pXest_comm_find_owner(::Type{Val{Dc}},ptr_pXest,itree,quad,guess) where Dc if (Dc==2) return p4est_comm_find_owner(ptr_pXest,itree,quad,guess) @@ -1681,39 +1562,6 @@ function _pXest_comm_find_owner(::Type{Val{Dc}},ptr_pXest,itree,quad,guess) wher end end -function _pXest_quadrant_array_index(::Type{Val{Dc}}, quadrants, iquad) where Dc - if (Dc==2) - return p4est_quadrant_array_index(quadrants, iquad) - elseif (Dc==3) - return p8est_quadrant_array_index(quadrants, iquad) - end -end - - -function _pXest_quadrant_is_equal(::Type{Val{Dc}}, q1, q2) where Dc - if (Dc==2) - return p4est_quadrant_is_equal(q1,q2) - elseif (Dc==3) - return p8est_quadrant_is_equal(q1, q2) - end -end - -function _pXest_quadrant_is_parent(::Type{Val{Dc}}, q1, q2) where Dc - if (Dc==2) - return p4est_quadrant_is_parent(q1,q2) - elseif (Dc==3) - return p8est_quadrant_is_parent(q1,q2) - end -end - -function _pXest_quadrant_compare(::Type{Val{Dc}}, q1, q2) where Dc - if (Dc==2) - return p4est_quadrant_compare(q1,q2) - elseif (Dc==3) - return p8est_quadrant_compare(q1,q2) - end -end - function _p4est_compute_migration_control_data(::Type{Val{Dc}},ptr_pXest_old,ptr_pXest_new) where Dc pXest_old = ptr_pXest_old[] pXest_new = ptr_pXest_new[] @@ -1724,12 +1572,14 @@ function _p4est_compute_migration_control_data(::Type{Val{Dc}},ptr_pXest_old,ptr old2new = Vector{Int}(undef,pXest_old.local_num_quadrants) current_old_quad_index = 1 + pXest_type = _dim_to_pXest_type(Dc) + for itree = 0:num_trees-1 - tree = _pXest_tree_array_index(Val{Dc},pXest_old.trees,itree)[] + tree = pXest_tree_array_index(pXest_type,pXest_old,itree)[] num_quads = Cint(tree.quadrants.elem_count) for iquad = 0:num_quads-1 - q = _pXest_quadrant_array_index(Val{Dc},tree.quadrants, iquad) + q = pXest_quadrant_array_index(pXest_type, tree, iquad) new_rank = _pXest_comm_find_owner(Val{Dc},ptr_pXest_new,itree,q,0) if (new_rank != my_rank) if (!(new_rank+1 in keys(ranks_count))) @@ -1740,9 +1590,9 @@ function _p4est_compute_migration_control_data(::Type{Val{Dc}},ptr_pXest_old,ptr old2new[current_old_quad_index] = 0 else current_new_quad_index = 1 - new_tree = _pXest_tree_array_index(Val{Dc},pXest_new.trees,pXest_new.first_local_tree)[] + new_tree = pXest_tree_array_index(pXest_type,pXest_new,pXest_new.first_local_tree)[] for t = pXest_new.first_local_tree:pXest_new.last_local_tree - new_tree = _pXest_tree_array_index(Val{Dc},pXest_new.trees,t)[] + new_tree = pXest_tree_array_index(pXest_type,pXest_new,t)[] if t == itree break end @@ -1751,8 +1601,8 @@ function _p4est_compute_migration_control_data(::Type{Val{Dc}},ptr_pXest_old,ptr found = false num_quads_new = Cint(new_tree.quadrants.elem_count) for iquad_new = 0:num_quads_new-1 - q_new = _pXest_quadrant_array_index(Val{Dc},new_tree.quadrants, iquad_new) - found = _pXest_quadrant_is_equal(Val{Dc},q,q_new)!=0 + q_new = pXest_quadrant_array_index(pXest_type, new_tree, iquad_new) + found = pXest_quadrant_is_equal(pXest_type,q,q_new) if found break end @@ -1839,8 +1689,8 @@ function _redistribute_parts_subseteq_parts_redistributed(model::OctreeDistribut model.parts.comm, parts.comm) end - ptr_pXest_new = pXest_copy(Val{Dc}, ptr_pXest_old) - pXest_partition!(Val{Dc}, ptr_pXest_new) + ptr_pXest_new = pXest_copy(model.pXest_type, ptr_pXest_old) + pXest_partition!(model.pXest_type, ptr_pXest_new) # Compute RedistributeGlue parts_snd, lids_snd, old2new = _p4est_compute_migration_control_data(Val{Dc},ptr_pXest_old,ptr_pXest_new) @@ -1856,7 +1706,7 @@ function _redistribute_parts_subseteq_parts_redistributed(model::OctreeDistribut ptr_pXest_lnodes = setup_pXest_lnodes_nonconforming(Val{Dc}, ptr_pXest_new, ptr_pXest_ghost) # Build fine-grid mesh - fmodel, non_conforming_glue = setup_non_conforming_distributed_discrete_model(Val{Dc}, + fmodel, non_conforming_glue = setup_non_conforming_distributed_discrete_model(model.pXest_type, parts, model.coarse_model, model.ptr_pXest_connectivity, @@ -1864,8 +1714,8 @@ function _redistribute_parts_subseteq_parts_redistributed(model::OctreeDistribut ptr_pXest_ghost, ptr_pXest_lnodes) - pXest_lnodes_destroy(Val{Dc},ptr_pXest_lnodes) - pXest_ghost_destroy(Val{Dc},ptr_pXest_ghost) + pXest_lnodes_destroy(model.pXest_type,ptr_pXest_lnodes) + pXest_ghost_destroy(model.pXest_type,ptr_pXest_ghost) red_model = OctreeDistributedDiscreteModel(Dc,Dp, parts, @@ -1874,6 +1724,7 @@ function _redistribute_parts_subseteq_parts_redistributed(model::OctreeDistribut model.coarse_model, model.ptr_pXest_connectivity, ptr_pXest_new, + model.pXest_type, false, model) return red_model, glue @@ -1909,7 +1760,7 @@ function _redistribute_parts_supset_parts_redistributed( end # p4est_vtk_write_file(model.ptr_pXest, C_NULL, "model.ptr_pXest") - ptr_pXest_old=pXest_copy(Val{Dc},model.ptr_pXest) + ptr_pXest_old=pXest_copy(model.pXest_type,model.ptr_pXest) pXest_partition_given!(Val{Dc}, ptr_pXest_old, num_cells_per_part) # p4est_vtk_write_file(ptr_pXest_old, C_NULL, "ptr_pXest_old") @@ -1928,7 +1779,7 @@ function _redistribute_parts_supset_parts_redistributed( parts_rcv, lids_rcv, new2old = _p4est_compute_migration_control_data(Val{Dc},ptr_pXest_old,model.ptr_pXest) - pXest_destroy(Val{Dc},ptr_pXest_old) + pXest_destroy(model.pXest_type,ptr_pXest_old) lids_rcv, parts_rcv, lids_snd, parts_snd, old2new, new2old = _to_pdata(model.parts, lids_rcv, parts_rcv, lids_snd, parts_snd, old2new, new2old) @@ -1953,8 +1804,10 @@ function _redistribute_parts_supset_parts_redistributed( ptr_pXest_ghost, ptr_pXest_lnodes) - pXest_lnodes_destroy(Val{Dc},ptr_pXest_lnodes) - pXest_ghost_destroy(Val{Dc},ptr_pXest_ghost) + pXest_type = _dim_to_pXest_type(Dc) + + pXest_lnodes_destroy(pXest_type,ptr_pXest_lnodes) + pXest_ghost_destroy(pXest_type,ptr_pXest_ghost) non_conforming_glue = _create_conforming_model_non_conforming_glue(fmodel) @@ -1965,6 +1818,7 @@ function _redistribute_parts_supset_parts_redistributed( model.coarse_model, model.ptr_pXest_connectivity, ptr_pXest_new, + pXest_type, false, model) return red_model, glue @@ -2014,7 +1868,15 @@ const p8est_face_corners = [ 0 2 4 6 ; 2 3 6 7 ; 0 1 2 3 ; 4 5 6 7 ] - + +const p6est_subface_to_hanging_edges_within_subface = +[ + 1; + 0; + 3; + 2; +] + const p8est_subface_to_hanging_edges_within_subface = [ 1 3; @@ -2023,6 +1885,14 @@ const p8est_subface_to_hanging_edges_within_subface = 0 2; ] +const p6est_subface_to_hanging_edges_within_face = +[ + 1; + 1; + 1; + 1; +] + const p8est_subface_to_hanging_edges_within_face = [ 3 1; @@ -2031,7 +1901,6 @@ const p8est_subface_to_hanging_edges_within_face = 4 2; ] - const p8est_edge_corners = [ 0 1; 2 3; 4 5; @@ -2046,7 +1915,6 @@ const p8est_edge_corners = [ 0 1; 3 7 ] - const hanging_vertex_code = -2 const hanging_edge_from_face_code = -3 function num_cell_vertices(::Type{Val{Dc}}) where Dc @@ -2061,121 +1929,35 @@ function num_cell_faces(::Type{Val{Dc}}) where Dc 2*Dc end - -# To add to P4est_wrapper.jl library -# I just translated this function to Julia from its p4est counterpart -# We cannot call it directly because it is declared as static within p4est, -# and thus it does not belong to the ABI of the dynamic library object. - -# /** Decode the face_code into hanging face information. -# * -# * This is mostly for demonstration purposes. Applications probably will -# * integrate it into their own loop over the face for performance reasons. -# * -# * \param[in] face_code as in the p4est_lnodes_t structure. -# * \param[out] hanging face: if there are hanging faces, -# * hanging_face = -1 if the face is not hanging, -# * = 0 if the face is the first half, -# * = 1 if the face is the second half. -# * note: not touched if there are no hanging faces. -# * \return true if any face is hanging, false otherwise. -# */ - -const p4est_corner_faces = [0 2; 1 2; 0 3; 1 3] -const p4est_corner_face_corners = [0 -1 0 -1; -1 0 1 -1; 1 -1 -1 0; -1 1 -1 1] -function p4est_lnodes_decode(face_code, hanging_face) - @assert face_code >= 0 - if (face_code != 0) - c = face_code & 0x03 - work = face_code >> 2 - hanging_face .= -1 - for i = 0:1 - f = p4est_corner_faces[c+1, i+1] - hanging_face[f+1] = (work & 0x01) != 0 ? p4est_corner_face_corners[c+1, f+1] : -1 - work >>= 1 - end - return 1 - else - return 0 - end -end - -const p8est_corner_faces = [0 2 4; 1 2 4; 0 3 4; 1 3 4; 0 2 5; 1 2 5; 0 3 5; 1 3 5] - -const p8est_face_edges = [ 4 6 8 10; 5 7 9 11; 0 2 8 9; 1 3 10 11; 0 1 4 5; 2 3 6 7] - -const p8est_corner_face_corners = [0 -1 0 -1 0 -1; -1 0 1 -1 1 -1 ; 1 -1 -1 0 2 -1 ; -1 1 -1 1 3 -1 ; - 2 -1 2 -1 -1 0; -1 2 3 -1 -1 1 ; 3 -1 -1 2 -1 2 ; -1 3 -1 3 -1 3 ] - -const p8est_corner_edges = [ 0 4 8; 0 5 9; 1 4 10; 1 5 11; 2 6 8; 2 7 9; 3 6 10; 3 7 11 ] - -# To add to p8est_wrapper.jl library -# I just translated this function to Julia from its p4est counterpart -# We cannot call it directly because it is declared as static within p4est, -# and thus it does not belong to the ABI of the dynamic library object. -function p8est_lnodes_decode(face_code, - hanging_face, - hanging_edge) - @assert face_code >= 0 - if (face_code!=0) - c = face_code & 0x0007 - work = face_code >> 3 - hanging_face .= -1 - hanging_edge .= -1 - cwork = c - for i=0:2 - if ((work & 0x0001)!=0) - f = p8est_corner_faces[c+1,i+1] - hanging_face[f+1] = p8est_corner_face_corners[c+1,f+1] - for j=0:3 - e = p8est_face_edges[f+1,j+1] - hanging_edge[e+1] = 4 - end - end - work >>= 1 - end - for i=0:3 - if ((work & 0x0001)!=0) - e = p8est_corner_edges[c+1,i+1] - hanging_edge[e+1] = (hanging_edge[e+1] == -1) ? 0 : 2 - hanging_edge[e+1] += (cwork & 0x0001) - end - cwork >>= 1 - work >>= 1 - end - return 1 - else - return 0 - end -end - -function setup_non_conforming_distributed_discrete_model(::Type{Val{Dc}}, +function setup_non_conforming_distributed_discrete_model(pXest_type::PXestType, parts, coarse_discrete_model, ptr_pXest_connectivity, ptr_pXest, ptr_pXest_ghost, - ptr_pXest_lnodes) where Dc + ptr_pXest_lnodes) - cell_prange = setup_cell_prange(Val{Dc}, parts, ptr_pXest, ptr_pXest_ghost) + Dc=num_cell_dims(pXest_type) + + cell_prange = setup_cell_prange(pXest_type, parts, ptr_pXest, ptr_pXest_ghost) gridap_cell_faces, non_conforming_glue= - generate_cell_faces_and_non_conforming_glue(Val{Dc},ptr_pXest_lnodes, cell_prange) + generate_cell_faces_and_non_conforming_glue(pXest_type,ptr_pXest_lnodes, cell_prange) nlvertices = map(non_conforming_glue) do ncglue ncglue.num_regular_faces[1]+ncglue.num_hanging_faces[1] end - node_coordinates=generate_node_coordinates(Val{Dc}, + node_coordinates=generate_node_coordinates(pXest_type, gridap_cell_faces[1], nlvertices, ptr_pXest_connectivity, ptr_pXest, ptr_pXest_ghost) - grid,topology=generate_grid_and_topology(Val{Dc}, + grid,topology=generate_grid_and_topology(pXest_type, gridap_cell_faces[1], nlvertices, node_coordinates) @@ -2194,14 +1976,17 @@ function setup_non_conforming_distributed_discrete_model(::Type{Val{Dc}}, end end - face_labeling=generate_face_labeling(parts, + face_labeling=generate_face_labeling(pXest_type, + parts, cell_prange, coarse_discrete_model, topology, ptr_pXest, ptr_pXest_ghost) - _set_hanging_labels!(face_labeling,non_conforming_glue) + coarse_face_labeling = get_face_labeling(coarse_discrete_model) + + _set_hanging_labels!(face_labeling,non_conforming_glue,coarse_face_labeling) discretemodel=map(grid,topology,face_labeling) do grid, topology, face_labeling Gridap.Geometry.UnstructuredDiscreteModel(grid,topology,face_labeling) @@ -2209,23 +1994,27 @@ function setup_non_conforming_distributed_discrete_model(::Type{Val{Dc}}, GridapDistributed.DistributedDiscreteModel(discretemodel,cell_prange), non_conforming_glue end -function _set_hanging_labels!(face_labeling,non_conforming_glue) - max_entity_ids = map(face_labeling) do face_labeling - max_entity_id = typemin(eltype(first(face_labeling.d_to_dface_to_entity))) - for i=1:length(face_labeling.d_to_dface_to_entity) - max_entity_id=max(maximum(face_labeling.d_to_dface_to_entity[i]),max_entity_id) - end - max_entity_id +function _compute_max_entity_id(face_labeling) + max_entity_id = typemin(eltype(first(face_labeling.d_to_dface_to_entity))) + for i=1:length(face_labeling.d_to_dface_to_entity) + max_entity_id=max(maximum(face_labeling.d_to_dface_to_entity[i]),max_entity_id) end - max_entity_id = reduction(max, - max_entity_ids, - destination=:all, - init=zero(eltype(max_entity_ids))) - + max_entity_id +end + +function _compute_min_entity_id(face_labeling) + min_entity_id = typemin(eltype(first(face_labeling.d_to_dface_to_entity))) + for i=1:length(face_labeling.d_to_dface_to_entity) + min_entity_id=min(minimum(face_labeling.d_to_dface_to_entity[i]),min_entity_id) + end + min_entity_id +end + +function _set_hanging_labels!(face_labeling,non_conforming_glue,coarse_face_labeling) + max_entity_id = _compute_max_entity_id(coarse_face_labeling) hanging_entitity_ids = Dict{Int,Bool}() - map(max_entity_id, - face_labeling, - non_conforming_glue) do max_entity_id,face_labeling,ncglue + map(face_labeling, + non_conforming_glue) do face_labeling,ncglue for i=1:length(ncglue.num_hanging_faces) num_regular_faces_i = ncglue.num_regular_faces[i] num_hanging_faces_i = ncglue.num_hanging_faces[i] @@ -2242,712 +2031,6 @@ function _set_hanging_labels!(face_labeling,non_conforming_glue) end end -function _build_map_from_faces_to_cell_lface(vnodes, element_nodes, face_code) - n_cell_faces = num_cell_faces(Val{2}) - hanging_face = Vector{Cint}(undef, n_cell_faces) - - # Build a map from faces to (cell,lface) - p4est_gface_to_gcell_p4est_lface = Dict{Int,Tuple{Int,Int}}() - for cell = 1:length(face_code) - start = (cell - 1) * vnodes + 1 - p4est_cell_faces = view(element_nodes, start:start+n_cell_faces-1) - has_hanging = p4est_lnodes_decode(face_code[cell], hanging_face) - if (has_hanging==0) - for (lface, gface) in enumerate(p4est_cell_faces) - p4est_gface_to_gcell_p4est_lface[gface] = (cell, lface) - end - else - for (lface, half) in enumerate(hanging_face) - # Current face is NOT hanging - if (half == -1) - gface = p4est_cell_faces[lface] - p4est_gface_to_gcell_p4est_lface[gface] = (cell, lface) - end - end - end - end - p4est_gface_to_gcell_p4est_lface -end - -function _build_map_from_faces_edges_to_cell_lface_ledge(vnodes, element_nodes, face_code) - n_cell_faces = num_cell_faces(Val{3}) - n_cell_edges = num_cell_edges(Val{3}) - - hanging_face = Vector{Cint}(undef, n_cell_faces) - hanging_edge = Vector{Cint}(undef, n_cell_edges) - - # Build a map from faces to (cell,lface) - p4est_gface_to_gcell_p4est_lface = Dict{Int,Tuple{Int,Int}}() - p4est_gedge_to_gcell_p4est_ledge = Dict{Int,Tuple{Int,Int}}() - for cell = 1:length(face_code) - start = (cell - 1) * vnodes + 1 - p4est_cell_faces = view(element_nodes, start:start+n_cell_faces-1) - p4est_cell_edges = view(element_nodes, start+n_cell_faces:start+n_cell_faces+n_cell_edges-1) - - - has_hanging = p8est_lnodes_decode(face_code[cell], hanging_face, hanging_edge) - if (has_hanging==0) - for (lface, gface) in enumerate(p4est_cell_faces) - p4est_gface_to_gcell_p4est_lface[gface] = (cell, lface) - end - for (ledge, gedge) in enumerate(p4est_cell_edges) - p4est_gedge_to_gcell_p4est_ledge[gedge] = (cell, ledge) - end - else - for (lface, half) in enumerate(hanging_face) - # Current face is NOT hanging - if (half == -1) - gface = p4est_cell_faces[lface] - p4est_gface_to_gcell_p4est_lface[gface] = (cell, lface) - end - end - for (ledge, half) in enumerate(hanging_edge) - # Current edge is NOT hanging - if (half == -1) - gedge = p4est_cell_edges[ledge] - p4est_gedge_to_gcell_p4est_ledge[gedge] = (cell, ledge) - end - end - end - end - p4est_gface_to_gcell_p4est_lface, p4est_gedge_to_gcell_p4est_ledge -end - -function pXest_2_gridap_vertex(::Type{Val{Dc}}) where Dc - Gridap.Arrays.IdentityVector(num_cell_vertices(Val{Dc})) -end - -function p8est_2_gridap_edge() - Gridap.Arrays.IdentityVector(num_cell_edges(Val{3})) -end -function pXest_2_gridap_facet(::Type{Val{Dc}}) where Dc - if (Dc==2) - GridapP4est.P4EST_2_GRIDAP_FACET_2D - else - @assert Dc==3 - GridapP4est.P4EST_2_GRIDAP_FACET_3D - end -end - -function hanging_lvertex_within_face_2d(half) - half == 0 ? 1 : 0 -end - -function hanging_lvertex_within_face_3d(half) - if (half==0) - return 3 - elseif (half==1) - return 2 - elseif (half==2) - return 1 - elseif (half==3) - return 0 - end -end - -function hanging_lvertex_within_edge(half) - if (half==0 || half==2) - return 1 - elseif (half==1 || half==3) - return 0 - end - @assert false -end - -function regular_lvertex_within_face(half) - return half -end - -function regular_lvertex_within_edge(half) - if (half==0 || half==2) - return 0 - elseif (half==1 || half==3) - return 1 - end - @assert false -end - - -function generate_cell_faces_and_non_conforming_glue(::Type{Val{Dc}}, - ptr_pXest_lnodes, - cell_prange) where Dc - - n_cell_vertices = num_cell_vertices(Val{Dc}) - n_cell_edges = num_cell_edges(Val{Dc}) - n_cell_faces = num_cell_faces(Val{Dc}) - - lnodes = ptr_pXest_lnodes[] - element_nodes = unsafe_wrap(Array, lnodes.element_nodes, lnodes.vnodes * lnodes.num_local_elements) - face_code = unsafe_wrap(Array, lnodes.face_code, lnodes.num_local_elements) - hanging_face = Vector{Cint}(undef, n_cell_faces) - face_code_with_ghosts = map(partition(cell_prange)) do indices - @assert length(face_code)==own_length(indices) - @assert own_length(indices)==lnodes.num_local_elements - face_code_with_ghosts=similar(face_code, local_length(indices)) - face_code_with_ghosts[1:own_length(indices)] .= face_code - face_code_with_ghosts - end - - cache_face_code=fetch_vector_ghost_values_cache(face_code_with_ghosts, partition(cell_prange)) - fetch_vector_ghost_values!(face_code_with_ghosts, cache_face_code) |> wait - - element_nodes_with_ghosts = map(partition(cell_prange)) do indices - nonlocal_nodes = unsafe_wrap(Array, lnodes.nonlocal_nodes, lnodes.num_local_nodes-lnodes.owned_count) - element_nodes_with_ghosts_data=similar(element_nodes, local_length(indices)*lnodes.vnodes) - for (i,node) in enumerate(element_nodes) - if (node wait - - map(element_nodes_with_ghosts,face_code_with_ghosts,partition(cell_prange)) do element_nodes_with_ghosts, face_code_with_ghosts, indices - @debug "ENDES[$(part_id(indices))]: $(element_nodes_with_ghosts.data)" - @debug "FCODS[$(part_id(indices))]: $(face_code_with_ghosts)" - end - - if (Dc==2) - hanging_lvertex_within_face=hanging_lvertex_within_face_2d - pXest_face_corners = p4est_face_corners - else - hanging_lvertex_within_face=hanging_lvertex_within_face_3d - pXest_face_corners = p8est_face_corners - end - - if (Dc==3) - hanging_edge = Vector{Cint}(undef, n_cell_edges) - end - - num_regular_faces, - num_hanging_faces, - gridap_cell_faces, - hanging_faces_glue = - map(partition(cell_prange), - element_nodes_with_ghosts, - face_code_with_ghosts) do indices, - element_nodes_with_ghosts, - face_code_with_ghosts - @assert local_length(indices)==length(face_code_with_ghosts) - - num_local_elements = local_length(indices) - num_regular_faces = Vector{Int}(undef, Dc) - num_hanging_faces = Vector{Int}(undef, Dc) - - # Count regular vertices - num_regular_faces[1] = 0 - regular_vertices_p4est_to_gridap = Dict{Int,Int}() - - num_regular_faces[Dc] = 0 - regular_faces_p4est_to_gridap = Dict{Int,Int}() - - if (Dc==2) - p4est_gface_to_gcell_p4est_lface = - _build_map_from_faces_to_cell_lface(lnodes.vnodes, element_nodes_with_ghosts.data, face_code_with_ghosts) - else - p4est_gface_to_gcell_p4est_lface, - p4est_gedge_to_gcell_p4est_ledge = - _build_map_from_faces_edges_to_cell_lface_ledge(lnodes.vnodes, - element_nodes_with_ghosts.data, - face_code_with_ghosts) - end - - PXEST_2_GRIDAP_VERTEX = pXest_2_gridap_vertex(Val{Dc}) - PXEST_2_GRIDAP_FACE = pXest_2_gridap_facet(Val{Dc}) - PXEST_2_GRIDAP_EDGE = p8est_2_gridap_edge() - - n = local_length(indices) - gridap_cell_vertices_ptrs = Vector{Int32}(undef,n+1) - gridap_cell_faces_ptrs = Vector{Int32}(undef,n+1) - gridap_cell_vertices_ptrs[1]=1 - gridap_cell_faces_ptrs[1]=1 - - hanging_vertices_pairs_to_owner_face = Dict{Tuple{Int,Int},Int}() - hanging_faces_pairs_to_owner_face = Dict{Tuple{Int,Int},Tuple{Int,Int}}() - - for i=1:n - gridap_cell_vertices_ptrs[i+1]=gridap_cell_vertices_ptrs[i]+n_cell_vertices - gridap_cell_faces_ptrs[i+1]=gridap_cell_faces_ptrs[i]+n_cell_faces - end - - gridap_cell_vertices_data = Vector{Int}(undef, num_local_elements * n_cell_vertices) - gridap_cell_vertices_data .= -1 - - gridap_cell_faces_data = Vector{Int}(undef, num_local_elements * n_cell_faces) - gridap_cell_faces_data .= -1 - - if (Dc==3) - num_regular_faces[2] = 0 - - gridap_cell_edges_ptrs = Vector{Int32}(undef,n+1) - gridap_cell_edges_ptrs[1]=1 - for i=1:n - gridap_cell_edges_ptrs[i+1]=gridap_cell_edges_ptrs[i]+n_cell_edges - end - gridap_cell_edges_data = Vector{Int}(undef, num_local_elements * n_cell_edges) - gridap_cell_edges_data .= -1 - hanging_edges_cell_ledge_to_owner_face_half = Dict{Tuple{Int,Int},Tuple{Int,Int}}() - owner_edge_subedge_to_cell_ledge = Dict{Tuple{Int,Int},Tuple{Int,Int}}() - hanging_vertices_pairs_to_owner_edge = Dict{Tuple{Int,Int},Int}() - regular_edges_p4est_to_gridap = Dict{Int,Int}() - end - - for cell = 1:num_local_elements - start = (cell - 1) * lnodes.vnodes + 1 - start_gridap_vertices = (cell - 1) * n_cell_vertices - start_gridap_faces = (cell - 1) * n_cell_faces - - p4est_cell_faces = view(element_nodes_with_ghosts.data, start:start+n_cell_faces-1) - p4est_cell_vertices = view(element_nodes_with_ghosts.data, - start+n_cell_faces+n_cell_edges:start+n_cell_faces+n_cell_edges+n_cell_vertices-1) - - gridap_cell_vertices = view(gridap_cell_vertices_data, - start_gridap_vertices+1:start_gridap_vertices+n_cell_vertices) - gridap_cell_faces = view(gridap_cell_faces_data, - start_gridap_faces+1:start_gridap_faces+n_cell_faces) - - if (Dc==2) - has_hanging = p4est_lnodes_decode(face_code_with_ghosts[cell], hanging_face) - else - has_hanging = p8est_lnodes_decode(face_code_with_ghosts[cell], hanging_face, hanging_edge) - start_gridap_edges = (cell-1)*n_cell_edges - gridap_cell_edges = view(gridap_cell_edges_data, start_gridap_edges+1:start_gridap_edges+n_cell_edges) - p4est_cell_edges = view(element_nodes_with_ghosts.data, - start+n_cell_faces:start+n_cell_faces+n_cell_edges-1) - end - if has_hanging == 0 - # All vertices/edges/faces of the current cell are regular - # Process vertices - for (p4est_lvertex, p4est_gvertex) in enumerate(p4est_cell_vertices) - num_regular_faces[1] = - process_current_face!(gridap_cell_vertices, - regular_vertices_p4est_to_gridap, - num_regular_faces[1], - p4est_cell_vertices, - p4est_lvertex, - p4est_gvertex, - PXEST_2_GRIDAP_VERTEX) - end - - if (Dc==3) - for (p4est_ledge, p4est_gedge) in enumerate(p4est_cell_edges) - num_regular_faces[2] = - process_current_face!(gridap_cell_edges, - regular_edges_p4est_to_gridap, - num_regular_faces[2], - p4est_cell_edges, - p4est_ledge, - p4est_gedge, - PXEST_2_GRIDAP_EDGE) - end - end - - # Process faces - for (p4est_lface, p4est_gface) in enumerate(p4est_cell_faces) - num_regular_faces[Dc] = - process_current_face!(gridap_cell_faces, - regular_faces_p4est_to_gridap, - num_regular_faces[Dc], - p4est_cell_faces, - p4est_lface, - p4est_gface, - PXEST_2_GRIDAP_FACE) - end - else - # "Touch" hanging vertices before processing current cell - # This is required as we dont have any means to detect - # a hanging vertex from a non-hanging face - for (p4est_lface, half) in enumerate(hanging_face) - if (half != -1) - hanging_vertex_lvertex_within_face = hanging_lvertex_within_face(half) - p4est_lvertex = pXest_face_corners[p4est_lface, - hanging_vertex_lvertex_within_face+1] - gridap_cell_vertices[PXEST_2_GRIDAP_VERTEX[p4est_lvertex+1]] = hanging_vertex_code - end - end - - if (Dc==3) - for (p4est_ledge, half) in enumerate(hanging_edge) - if (half != -1 && half !=4) - hanging_vertex_lvertex_within_edge = hanging_lvertex_within_edge(half) - p4est_lvertex = p8est_edge_corners[p4est_ledge, - hanging_vertex_lvertex_within_edge+1] - gridap_cell_vertices[PXEST_2_GRIDAP_VERTEX[p4est_lvertex+1]] = hanging_vertex_code - end - end - end - - # Current cell has at least one hanging face - for (p4est_lface, half) in enumerate(hanging_face) - # Current face is NOT hanging - if (half == -1) - # Process vertices on the boundary of p4est_lface - for p4est_lvertex in pXest_face_corners[p4est_lface, :] - p4est_gvertex = p4est_cell_vertices[p4est_lvertex+1] - if (gridap_cell_vertices[p4est_lvertex+1] != hanging_vertex_code) - num_regular_faces[1] = - process_current_face!(gridap_cell_vertices, - regular_vertices_p4est_to_gridap, - num_regular_faces[1], - p4est_cell_vertices, - p4est_lvertex + 1, - p4est_gvertex, - PXEST_2_GRIDAP_VERTEX) - end - end - # Process non-hanging face - p4est_gface = p4est_cell_faces[p4est_lface] - num_regular_faces[Dc] = - process_current_face!(gridap_cell_faces, - regular_faces_p4est_to_gridap, - num_regular_faces[Dc], - p4est_cell_faces, - p4est_lface, - p4est_gface, - PXEST_2_GRIDAP_FACE) - else # Current face is hanging - # Identify regular vertex and hanging vertex - # Repeat code above for regular vertex - # Special treatment for hanging vertex - regular_vertex_lvertex_within_face = regular_lvertex_within_face(half) - hanging_vertex_lvertex_within_face = hanging_lvertex_within_face(half) - - # Process regular vertex - p4est_regular_lvertex = pXest_face_corners[p4est_lface, regular_vertex_lvertex_within_face+1] - p4est_gvertex = p4est_cell_vertices[p4est_regular_lvertex+1] - num_regular_faces[1] = - process_current_face!(gridap_cell_vertices, - regular_vertices_p4est_to_gridap, - num_regular_faces[1], - p4est_cell_vertices, - p4est_regular_lvertex + 1, - p4est_gvertex, - PXEST_2_GRIDAP_VERTEX) - - # Process hanging vertex - p4est_hanging_lvertex = pXest_face_corners[p4est_lface, hanging_vertex_lvertex_within_face+1] - owner_face = p4est_cell_faces[p4est_lface] - hanging_vertices_pairs_to_owner_face[(cell, PXEST_2_GRIDAP_VERTEX[p4est_hanging_lvertex+1])] = owner_face - - # Process hanging face - hanging_faces_pairs_to_owner_face[(cell, PXEST_2_GRIDAP_FACE[p4est_lface])] = (owner_face,half+1) - - if (Dc==3) - for (i,ledge_within_face) in enumerate(p8est_subface_to_hanging_edges_within_subface[half+1,:]) - p4est_ledge=p8est_face_edges[p4est_lface,ledge_within_face+1] - gridap_ledge = PXEST_2_GRIDAP_EDGE[p4est_ledge+1] - # Identify the two edges which are hanging within the face - hanging_edges_cell_ledge_to_owner_face_half[(cell, gridap_ledge)] = - (owner_face,-p8est_subface_to_hanging_edges_within_face[half+1,i]) - gridap_cell_edges[gridap_ledge] = hanging_edge_from_face_code - end - end - - end - end - - - if (Dc==3) - for (p4est_ledge, half) in enumerate(hanging_edge) - # Current edge is NOT hanging - if (half == -1) - # Process vertices on the boundary of p4est_ledge - for p4est_lvertex in p8est_edge_corners[p4est_ledge, :] - p4est_gvertex = p4est_cell_vertices[p4est_lvertex+1] - if (gridap_cell_vertices[p4est_lvertex+1] != hanging_vertex_code) - num_regular_faces[1] = - process_current_face!(gridap_cell_vertices, - regular_vertices_p4est_to_gridap, - num_regular_faces[1], - p4est_cell_vertices, - p4est_lvertex + 1, - p4est_gvertex, - PXEST_2_GRIDAP_VERTEX) - end - end - # Process non-hanging edge - p4est_gedge = p4est_cell_edges[p4est_ledge] - num_regular_faces[2] = - process_current_face!(gridap_cell_edges, - regular_edges_p4est_to_gridap, - num_regular_faces[2], - p4est_cell_edges, - p4est_ledge, - p4est_gedge, - PXEST_2_GRIDAP_EDGE) - else # Current edge is hanging - if ( gridap_cell_edges[PXEST_2_GRIDAP_EDGE[p4est_ledge]] != hanging_edge_from_face_code ) - # The present hanging edge cannot be within a coarser face - @assert half != 4 - - # # Identify regular vertex and hanging vertex - # # Repeat code above for regular vertex - # # Special treatment for hanging vertex - regular_vertex_lvertex_within_edge = regular_lvertex_within_edge(half) - hanging_vertex_lvertex_within_edge = hanging_lvertex_within_edge(half) - - # # Process regular vertex - p4est_regular_lvertex = p8est_edge_corners[p4est_ledge, regular_vertex_lvertex_within_edge+1] - p4est_gvertex = p4est_cell_vertices[p4est_regular_lvertex+1] - - num_regular_faces[1] = - process_current_face!(gridap_cell_vertices, - regular_vertices_p4est_to_gridap, - num_regular_faces[1], - p4est_cell_vertices, - p4est_regular_lvertex + 1, - p4est_gvertex, - PXEST_2_GRIDAP_VERTEX) - - # Process hanging vertex - p4est_hanging_lvertex = p8est_edge_corners[p4est_ledge, hanging_vertex_lvertex_within_edge+1] - p4est_owner_edge = p4est_cell_edges[p4est_ledge] - hanging_vertices_pairs_to_owner_edge[(cell, - PXEST_2_GRIDAP_VERTEX[p4est_hanging_lvertex+1])] = p4est_owner_edge - - # Process hanging edge - subedge = regular_vertex_lvertex_within_edge+1 - owner_edge_subedge_pair=(p4est_owner_edge,subedge) - gridap_ledge=PXEST_2_GRIDAP_EDGE[p4est_ledge] - hanging_edges_cell_ledge_to_owner_face_half[(cell, gridap_ledge)] = owner_edge_subedge_pair - if (!haskey(owner_edge_subedge_to_cell_ledge,owner_edge_subedge_pair)) - owner_edge_subedge_to_cell_ledge[owner_edge_subedge_pair] = (cell,gridap_ledge) - end - end - end - end - end - end - end - - function is_ghost(cell) - cell>own_length(indices) - end - - # Go over all touched hanging faces and start - # assigning IDs from the last num_regular_faces ID - # For each hanging face, keep track of (owner_cell,lface) - # Go over all hanging faces - # Detect if the owner face is in a ghost cell. - # If not in a ghost cell or touched - # Else - # The current face becomes a regular face - # end - hanging_faces_owner_cell_and_lface = - Vector{Tuple{Int,Int,Int}}(undef, length(keys(hanging_faces_pairs_to_owner_face))) - num_hanging_faces[Dc] = 0 - for key in keys(hanging_faces_pairs_to_owner_face) - (cell, lface) = key - (owner_p4est_gface, half) = hanging_faces_pairs_to_owner_face[key] - num_hanging_faces[Dc] += 1 - start_gridap_faces = (cell - 1) * n_cell_faces - gridap_cell_faces_data[start_gridap_faces+lface] = num_regular_faces[Dc] + num_hanging_faces[Dc] - if (!(is_ghost(cell)) || haskey(regular_faces_p4est_to_gridap,owner_p4est_gface)) - @assert haskey(regular_faces_p4est_to_gridap,owner_p4est_gface) - (owner_cell, p4est_lface) = p4est_gface_to_gcell_p4est_lface[owner_p4est_gface] - hanging_faces_owner_cell_and_lface[num_hanging_faces[Dc]] = - (owner_cell, n_cell_vertices+n_cell_edges+PXEST_2_GRIDAP_FACE[p4est_lface], half) - else - # Glue info cannot be computed for this hanging face - hanging_faces_owner_cell_and_lface[num_hanging_faces[Dc]] = (-1,-1,-1) - end - end - - @debug "[$(MPI.Comm_rank(MPI.COMM_WORLD))] gridap_cell_faces_data: $(gridap_cell_faces_data)" - - - # Go over all touched hanging vertices and start - # assigning IDs from the last num_regular_vertices ID - # For each hanging vertex, keep track of (owner_cell,lface) - num_hanging_faces[1] = 0 - hanging_vertices_owner_cell_and_lface = Tuple{Int,Int,Int}[] - half=1 - owner_p4est_gface_to_hanging_vertex = Dict{Int,Int}() - for key in keys(hanging_vertices_pairs_to_owner_face) - (cell, lvertex) = key - owner_p4est_gface = hanging_vertices_pairs_to_owner_face[key] - if !(haskey(owner_p4est_gface_to_hanging_vertex, owner_p4est_gface)) - num_hanging_faces[1] += 1 - owner_p4est_gface_to_hanging_vertex[owner_p4est_gface] = num_hanging_faces[1] - if (!is_ghost(cell) || (haskey(regular_faces_p4est_to_gridap,owner_p4est_gface))) - (owner_cell, p4est_lface) = p4est_gface_to_gcell_p4est_lface[owner_p4est_gface] - push!(hanging_vertices_owner_cell_and_lface, - (owner_cell, n_cell_vertices+n_cell_edges+PXEST_2_GRIDAP_FACE[p4est_lface],half)) - else - push!(hanging_vertices_owner_cell_and_lface,(-1, -1,-1)) - end - end - start_gridap_vertices = (cell - 1) * n_cell_vertices - gridap_cell_vertices_data[start_gridap_vertices+lvertex] = num_regular_faces[1] + - owner_p4est_gface_to_hanging_vertex[owner_p4est_gface] - end - - @debug "[$(MPI.Comm_rank(MPI.COMM_WORLD))] gridap_cell_vertices_data: $(gridap_cell_vertices_data)" - - if (Dc==3) - half=1 - owner_p4est_gedge_to_hanging_vertex = Dict{Int,Int}() - for key in keys(hanging_vertices_pairs_to_owner_edge) - (cell, lvertex) = key - owner_p4est_gedge = hanging_vertices_pairs_to_owner_edge[key] - if !(haskey(owner_p4est_gedge_to_hanging_vertex, owner_p4est_gedge)) - num_hanging_faces[1] += 1 - owner_p4est_gedge_to_hanging_vertex[owner_p4est_gedge] = num_hanging_faces[1] - if (!is_ghost(cell) || (haskey(regular_edges_p4est_to_gridap,owner_p4est_gedge))) - (owner_cell, p4est_ledge) = p4est_gedge_to_gcell_p4est_ledge[owner_p4est_gedge] - push!(hanging_vertices_owner_cell_and_lface, - (owner_cell, n_cell_vertices+PXEST_2_GRIDAP_EDGE[p4est_ledge],half)) - else - push!(hanging_vertices_owner_cell_and_lface,(-1, -1,-1)) - end - end - start_gridap_vertices = (cell - 1) * n_cell_vertices - gridap_cell_vertices_data[start_gridap_vertices+lvertex] = num_regular_faces[1] + - owner_p4est_gedge_to_hanging_vertex[owner_p4est_gedge] - end - - - # Go over all touched hanging edges and start - # assigning IDs from the last num_regular_edge ID - # For each hanging edge, keep track of (owner_cell,lface/ledge) - hanging_edges_owner_cell_and_lface = Tuple{Int,Int,Int}[] - owner_p4est_gface_half_to_hanging_edge = Dict{Tuple{Int,Int},Int}() - owner_p4est_gedge_subedge_to_hanging_edge = Dict{Tuple{Int,Int},Int}() - num_hanging_faces[2] = 0 - ledge_to_cvertices = Gridap.ReferenceFEs.get_faces(HEX, 1, 0) - # The following loop needs (1) the pairs to be traversed in increased order by cell ID; - # (2) gridap cell vertices to be already completed - for key in sort(collect(keys(hanging_edges_cell_ledge_to_owner_face_half))) - (cell, ledge) = key - (owner_p4est_gface_or_gedge, half) = hanging_edges_cell_ledge_to_owner_face_half[key] - @debug "[$(MPI.Comm_rank(MPI.COMM_WORLD))] own_length=$(own_length(indices)) cell=$(cell) ledge=$(ledge) owner_p4est_gface_or_gedge=$(owner_p4est_gface_or_gedge) half=$(half)" - if (half<0) # hanging edge is within a coarser face - owner_p4est_gface = owner_p4est_gface_or_gedge - if !(haskey(owner_p4est_gface_half_to_hanging_edge, (owner_p4est_gface,half))) - num_hanging_faces[2] += 1 - owner_p4est_gface_half_to_hanging_edge[(owner_p4est_gface,half)] = num_hanging_faces[2] - if (!is_ghost(cell) || (haskey(regular_faces_p4est_to_gridap,owner_p4est_gface))) - (owner_cell, p4est_lface) = p4est_gface_to_gcell_p4est_lface[owner_p4est_gface] - push!(hanging_edges_owner_cell_and_lface, - (owner_cell, n_cell_vertices+n_cell_edges+PXEST_2_GRIDAP_FACE[p4est_lface],half)) - else - push!(hanging_edges_owner_cell_and_lface,(-1, -1, -1)) - end - end - start_gridap_edges = (cell - 1) * n_cell_edges - gridap_cell_edges_data[start_gridap_edges+ledge] = num_regular_faces[2] + - owner_p4est_gface_half_to_hanging_edge[(owner_p4est_gface,half)] - - else # hanging edge is within a coarser edge - @assert half==1 || half==2 - owner_p4est_gedge = owner_p4est_gface_or_gedge - owner_gedge_pair = (owner_p4est_gedge,half) - if (haskey(owner_edge_subedge_to_cell_ledge,owner_gedge_pair)) - (owner_cell, owner_cell_ledge) = owner_edge_subedge_to_cell_ledge[owner_gedge_pair] - if (owner_cell==cell) - @assert owner_cell_ledge == ledge - num_hanging_faces[2] += 1 - owner_p4est_gedge_subedge_to_hanging_edge[(owner_p4est_gedge,half)] = num_hanging_faces[2] - if (!is_ghost(cell) || (haskey(regular_edges_p4est_to_gridap,owner_p4est_gedge))) - (owner_cell, p4est_ledge) = p4est_gedge_to_gcell_p4est_ledge[owner_p4est_gedge] - push!(hanging_edges_owner_cell_and_lface, - (owner_cell, n_cell_vertices+PXEST_2_GRIDAP_EDGE[p4est_ledge],half)) - else - push!(hanging_edges_owner_cell_and_lface,(-1, -1, -1)) - end - start_gridap_edges = (cell - 1) * n_cell_edges - gridap_cell_edges_data[start_gridap_edges+ledge] = num_regular_faces[2] + num_hanging_faces[2] - else - haskey_first_subedge = haskey(owner_p4est_gedge_subedge_to_hanging_edge,(owner_p4est_gedge,1)) - haskey_second_subedge = haskey(owner_p4est_gedge_subedge_to_hanging_edge,(owner_p4est_gedge,2)) - if (!(is_ghost(cell))) - @assert haskey_first_subedge && haskey_second_subedge - else - @assert haskey_first_subedge || haskey_second_subedge - end - if (haskey_first_subedge && haskey_second_subedge) - # The following code is required as we may have edges - # with different orientations at the inter-octree boundaries - match=true - start_gridap_vertices_cell = (cell - 1) * n_cell_vertices - start_gridap_vertices_cell_owner = (owner_cell - 1) * n_cell_vertices - for lvertex_cell in ledge_to_cvertices[ledge] - vertex_cell=gridap_cell_vertices_data[start_gridap_vertices_cell+lvertex_cell] - found=false - # Go over vertices of owner_cell_ledge in owner_cell - for lvertex_owner_cell in ledge_to_cvertices[owner_cell_ledge] - vertex_owner_cell=gridap_cell_vertices_data[start_gridap_vertices_cell_owner+lvertex_owner_cell] - if (vertex_owner_cell==vertex_cell) - found=true - break - end - end - if (!found) - match=false - break - end - end - if (match) - owner_half=half - else - owner_half=half==1 ? 2 : 1 - end - elseif (haskey_first_subedge) - owner_half=1 - elseif (haskey_second_subedge) - owner_half=2 - end - @debug "[$(MPI.Comm_rank(MPI.COMM_WORLD))] cell=$(cell) ledge=$(ledge) owner_p4est_gface_or_gedge=$(owner_p4est_gface_or_gedge) half=$(half) owner_half=$(owner_half)" - start_gridap_edges = (cell - 1) * n_cell_edges - gridap_cell_edges_data[start_gridap_edges+ledge] = - num_regular_faces[2] + owner_p4est_gedge_subedge_to_hanging_edge[(owner_p4est_gedge,owner_half)] - end - end - end - end - @debug "[$(MPI.Comm_rank(MPI.COMM_WORLD))] gridap_cell_edges_data: $(gridap_cell_edges_data)" - end - - - gridap_cell_faces = Vector{JaggedArray}(undef,Dc) - gridap_cell_faces[1] = JaggedArray(gridap_cell_vertices_data,gridap_cell_vertices_ptrs) - if (Dc==3) - gridap_cell_faces[2] = JaggedArray(gridap_cell_edges_data,gridap_cell_edges_ptrs) - end - gridap_cell_faces[Dc] = JaggedArray(gridap_cell_faces_data,gridap_cell_faces_ptrs) - - hanging_faces_glue = Vector{Vector{Tuple}}(undef,Dc) - hanging_faces_glue[1] = hanging_vertices_owner_cell_and_lface - if (Dc==3) - hanging_faces_glue[2] = hanging_edges_owner_cell_and_lface - end - hanging_faces_glue[Dc] = hanging_faces_owner_cell_and_lface - - - return num_regular_faces, - num_hanging_faces, - gridap_cell_faces, - hanging_faces_glue - - end |> tuple_of_arrays - - - gridap_cell_faces_out = Vector{MPIArray}(undef,Dc) - for i=1:Dc - gridap_cell_faces_out[i] = map(gridap_cell_faces) do gridap_cell_faces - gridap_cell_faces[i] - end - end - non_conforming_glue=map(num_regular_faces,num_hanging_faces,hanging_faces_glue) do nrf, nhf, hfg - NonConformingGlue(nrf, nhf, hfg) - end - gridap_cell_faces_out,non_conforming_glue - end diff --git a/src/PXestTypeMethods.jl b/src/PXestTypeMethods.jl new file mode 100644 index 0000000..c59bc5c --- /dev/null +++ b/src/PXestTypeMethods.jl @@ -0,0 +1,1927 @@ +abstract type PXestType end; +struct P4estType <: PXestType end ; +struct P6estType <: PXestType end; +struct P8estType <: PXestType end; +const P4P8estType = Union{P4estType,P8estType} +const P6P8estType = Union{P6estType,P8estType} + +abstract type PXestRefinementRuleType end; +struct PXestUniformRefinementRuleType <: PXestRefinementRuleType end; +struct PXestVerticalRefinementRuleType <: PXestRefinementRuleType end; +struct PXestHorizontalRefinementRuleType <: PXestRefinementRuleType end; + + + +function pXest_destroy(pXest_type::P4estType, ptr_pXest) + p4est_destroy(ptr_pXest) +end + +function pXest_destroy(pXest_type::P6estType, ptr_pXest) + p6est_destroy(ptr_pXest) +end + +function pXest_destroy(pXest_type::P8estType, ptr_pXest) + p8est_destroy(ptr_pXest) +end + + +function pXest_lnodes_destroy(pXest_type::P4estType, ptr_pXest_lnodes) + p4est_lnodes_destroy(ptr_pXest_lnodes) +end + +function pXest_lnodes_destroy(pXest_type::P6P8estType, ptr_pXest_lnodes) + p8est_lnodes_destroy(ptr_pXest_lnodes) +end + +function pXest_ghost_destroy(pXest_type::P4estType,ptr_pXest_ghost) + p4est_ghost_destroy(ptr_pXest_ghost) +end + +function pXest_ghost_destroy(pXest_type::P6estType,ptr_pXest_ghost) + p6est_ghost_destroy(ptr_pXest_ghost) +end + +function pXest_ghost_destroy(pXest_type::P8estType,ptr_pXest_ghost) + p8est_ghost_destroy(ptr_pXest_ghost) +end + +function pXest_connectivity_destroy(pXest_type::P4estType, ptr_pXest_connectivity) + p4est_connectivity_destroy(ptr_pXest_connectivity) +end + +function pXest_connectivity_destroy(pXest_type::P6estType, ptr_pXest_connectivity) + p6est_connectivity_destroy(ptr_pXest_connectivity) +end + +function pXest_connectivity_destroy(pXest_type::P8estType, ptr_pXest_connectivity) + p8est_connectivity_destroy(ptr_pXest_connectivity) +end + +function setup_pXest(pXest_type::P4estType, comm, connectivity, num_uniform_refinements) + p4est_new_ext(comm, + connectivity, + Cint(0), + Cint(num_uniform_refinements), + Cint(1), + Cint(0), + C_NULL, + C_NULL) +end + +function setup_pXest(pXest_type::P6estType, comm, connectivity, num_uniform_refinements) + p6est_new_ext(comm, + connectivity, + Cint(0), + Cint(num_uniform_refinements), # min_level + Cint(num_uniform_refinements), # min_zlevel + Cint(1), # num_zroot + Cint(0), # fill_uniform + Cint(1), # data_size + C_NULL, # init_fn + C_NULL) # user_pointer +end + +function setup_pXest(pXest_type::P6estType, + comm, connectivity, + num_horizontal_uniform_refinements, + num_vertical_uniform_refinements) + p6est_new_ext(comm, + connectivity, + Cint(0), + Cint(num_horizontal_uniform_refinements), # min_level + Cint(num_vertical_uniform_refinements), # min_zlevel + Cint(1), # num_zroot + Cint(0), # fill_uniform + Cint(1), # data_size + C_NULL, # init_fn + C_NULL) # user_pointer +end + +function setup_pXest(pXest_type::P8estType, comm, connectivity, num_uniform_refinements) + p8est_new_ext(comm, + connectivity, + Cint(0), Cint(num_uniform_refinements), Cint(1), Cint(0), + C_NULL, C_NULL) +end + +function setup_pXest_ghost(pXest_type::P4estType, ptr_pXest) + p4est_ghost_new(ptr_pXest,P4est_wrapper.P4EST_CONNECT_FULL) +end + +function setup_pXest_ghost(pXest_type::P6estType, ptr_pXest) + p6est_ghost_new(ptr_pXest,P4est_wrapper.P4EST_CONNECT_FULL) +end + +function setup_pXest_ghost(pXest_type::P8estType, ptr_pXest) + p8est_ghost_new(ptr_pXest,P4est_wrapper.P4EST_CONNECT_FULL) +end + +function setup_pXest_lnodes(pXest_type::P4estType, ptr_pXest, ptr_pXest_ghost) + p4est_lnodes_new(ptr_pXest, ptr_pXest_ghost, Cint(1)) +end + +function setup_pXest_lnodes(pXest_type::P8estType, ptr_pXest, ptr_pXest_ghost) + p8est_lnodes_new(ptr_pXest, ptr_pXest_ghost, Cint(1)) +end + +function setup_pXest_lnodes_nonconforming(pXest_type::P4estType, ptr_pXest, ptr_pXest_ghost) + p4est_lnodes_new(ptr_pXest, ptr_pXest_ghost, Cint(-2)) +end + +function setup_pXest_lnodes_nonconforming(pXest_type::P6estType, ptr_pXest, ptr_pXest_ghost) + p6est_lnodes_new(ptr_pXest, ptr_pXest_ghost, Cint(2)) +end + +function setup_pXest_lnodes_nonconforming(pXest_type::P8estType, ptr_pXest, ptr_pXest_ghost) + p8est_lnodes_new(ptr_pXest, ptr_pXest_ghost, Cint(-3)) +end + +function fill_tree_to_vertex!(conn,trian::Triangulation{D,D}) where D + cell_nodes_ids=Gridap.Geometry.get_cell_node_ids(trian) + tree_to_vertex=unsafe_wrap(Array, conn.tree_to_vertex, length(cell_nodes_ids)*(2^D)) + c=Gridap.Arrays.array_cache(cell_nodes_ids) + current=1 + for j=1:length(cell_nodes_ids) + ids=Gridap.Arrays.getindex!(c,cell_nodes_ids,j) + for id in ids + tree_to_vertex[current]=p4est_topidx_t(id-1) + current=current+1 + end + end +end + +function fill_coordinates!(conn,trian::Triangulation{D,D}) where D + node_coordinates=Gridap.Geometry.get_node_coordinates(trian) + vertices=unsafe_wrap(Array, conn.vertices, length(node_coordinates)*3) + current=1 + for i=1:length(node_coordinates) + p=node_coordinates[i] + for j=1:D + vertices[current]=Cdouble(p[j]) + current=current+1 + end + if (D==2) + vertices[current]=Cdouble(0.0) # Z coordinate always to 0.0 in 2D + current=current+1 + end + end +end + +function fill_tree_to_tree_and_to_face!(conn,trian::Triangulation{D,D}) where D + # /* + # * Fill tree_to_tree and tree_to_face to make sure we have a valid + # * connectivity. + # */ + PXEST_FACES=2*D + tree_to_tree=unsafe_wrap(Array, conn.tree_to_tree, conn.num_trees*PXEST_FACES ) + tree_to_face=unsafe_wrap(Array, conn.tree_to_face, conn.num_trees*PXEST_FACES ) + for tree=1:conn.num_trees + for face=1:PXEST_FACES + tree_to_tree[PXEST_FACES * (tree-1) + face] = tree-1 + tree_to_face[PXEST_FACES * (tree-1) + face] = face-1 + end + end +end + +function setup_pXest_connectivity(pXest_type::P4estType, coarse_discrete_model::DiscreteModel{2,2}) + trian=Triangulation(coarse_discrete_model) + pconn=p4est_connectivity_new( + p4est_topidx_t(num_nodes(trian)), # num_vertices + p4est_topidx_t(num_cells(coarse_discrete_model)), # num_trees + p4est_topidx_t(0), + p4est_topidx_t(0)) + conn=pconn[] + fill_tree_to_vertex!(conn, trian) + fill_coordinates!(conn, trian) + fill_tree_to_tree_and_to_face!(conn, trian) + p4est_connectivity_complete(pconn) + @assert Bool(p4est_connectivity_is_valid(pconn)) + pconn +end + +function setup_pXest_connectivity(pXest_type::P6estType, + coarse_discrete_model::DiscreteModel{2,2}, + extrusion_vector::Vector{Float64}) + @assert length(extrusion_vector)==3 + pconn4=setup_pXest_connectivity(P4estType(), coarse_discrete_model) + p6est_connectivity_new(pconn4,C_NULL,extrusion_vector) +end + +function setup_pXest_connectivity(pXest_type::P8estType, coarse_discrete_model::DiscreteModel{3,3}) + trian=Triangulation(coarse_discrete_model) + pconn=p8est_connectivity_new( + p4est_topidx_t(length(node_coordinates)), # num_vertices + p4est_topidx_t(num_cells(coarse_discrete_model)), # num_trees + p4est_topidx_t(0), + p4est_topidx_t(0), + p4est_topidx_t(0), + p4est_topidx_t(0)) + conn=pconn[] + fill_tree_to_vertex!(conn, trian) + fill_coordinates!(conn, trian) + fill_tree_to_tree_and_to_face!(conn, trian) + p8est_connectivity_complete(pconn) + @assert Bool(p8est_connectivity_is_valid(pconn)) + pconn +end + +function pXest_reset_data!(::P4estType, ptr_pXest, data_size, init_fn_c, user_pointer) + p4est_reset_data(ptr_pXest, data_size, init_fn_c, user_pointer) +end + +function pXest_reset_data!(::P6estType, ptr_pXest, data_size, init_fn_c, user_pointer) + p6est_reset_data(ptr_pXest, data_size, init_fn_c, user_pointer) +end + +function pXest_reset_data!(::P8estType, ptr_pXest, data_size, init_fn_c, user_pointer) + p8est_reset_data(ptr_pXest, data_size, init_fn_c, user_pointer) +end + + +function pXest_refine!(::P4estType, ptr_pXest, refine_fn_c, refine_replace_fn_c; init_fn_c=C_NULL) + p4est_refine_ext(ptr_pXest, Cint(0), Cint(-1), refine_fn_c, init_fn_c, refine_replace_fn_c) +end + +function pXest_refine!(::P8estType, ptr_pXest, refine_fn_c, refine_replace_fn_c; init_fn_c=C_NULL) + p8est_refine_ext(ptr_pXest, Cint(0), Cint(-1), refine_fn_c, init_fn_c, refine_replace_fn_c) +end + +function p6est_vertically_refine!(ptr_pXest, refine_fn_c, refine_replace_fn_c; init_fn_c=C_NULL) + p6est_refine_layers_ext(ptr_pXest, Cint(0), Cint(-1), refine_fn_c, init_fn_c, refine_replace_fn_c) +end + +function pXest_coarsen!(::P4estType, ptr_pXest, coarsen_fn_c) + p4est_coarsen(ptr_pXest, Cint(0), coarsen_fn_c, C_NULL) +end + +function pXest_coarsen!(::P8estType, ptr_pXest, coarsen_fn_c) + p8est_coarsen(ptr_pXest, Cint(0), coarsen_fn_c, C_NULL) +end + +function pXest_copy(::P4estType, ptr_pXest) + p4est_copy(ptr_pXest, Cint(0)) +end + +function pXest_copy(::P6estType, ptr_pXest) + p6est_copy(ptr_pXest, Cint(0)) +end + +function pXest_copy(::P8estType, ptr_pXest) + p8est_copy(ptr_pXest, Cint(0)) +end + +function pXest_balance!(::P4estType, ptr_pXest; k_2_1_balance=0) + if (k_2_1_balance==0) + p4est_balance(ptr_pXest, P4est_wrapper.P4EST_CONNECT_FULL, C_NULL) + else + @assert k_2_1_balance==1 + p4est_balance(ptr_pXest, P4est_wrapper.P4EST_CONNECT_FACE, C_NULL) + end +end + +function pXest_balance!(::P6estType, ptr_pXest; k_2_1_balance=0) + @assert k_2_1_balance==0 or k_2_1_balance==2 + if (k_2_1_balance==0) + p6est_balance(ptr_pXest, P4est_wrapper.P8EST_CONNECT_FULL, C_NULL) + elseif (k_2_1_balance==2) + p6est_balance(ptr_pXest, P4est_wrapper.P8EST_CONNECT_FACE, C_NULL) + end +end + +function pXest_balance!(::P8estType, ptr_pXest; k_2_1_balance=0) + if (k_2_1_balance==0) + p8est_balance(ptr_pXest, P4est_wrapper.P8EST_CONNECT_FULL, C_NULL) + elseif (k_2_1_balance==1) + p8est_balance(ptr_pXest, P4est_wrapper.P8EST_CONNECT_EDGE, C_NULL) + else + @assert k_2_1_balance==2 + p8est_balance(ptr_pXest, P4est_wrapper.P8EST_CONNECT_FACE, C_NULL) + end +end + +function pXest_partition!(::P4estType, ptr_pXest) + p4est_partition(ptr_pXest, 0, C_NULL) +end + +function pXest_partition!(::P6estType, ptr_pXest) + p6est_partition(ptr_pXest, 0, C_NULL) +end + +function pXest_partition!(::P8estType, ptr_pXest) + p8est_partition(ptr_pXest, 0, C_NULL) +end + + +function pXest_reset_callbacks(::P4estType) + # Variables which are updated accross calls to init_fn_callback + current_quadrant_index_within_tree = Cint(0) + current_quadrant_index_among_trees = Cint(0) + + # This C callback function is called once per quadtree quadrant. Here we are assuming + # that p4est->user_pointer has been set prior to the first call to this call + # back function to an array of ints with as many entries as forest quadrants. This call back function + # initializes the quadrant->p.user_data void * pointer of all quadrants such that it + # points to the corresponding entry in the global array mentioned in the previous sentence + function init_fn_callback(forest_ptr::Ptr{p4est_t}, + which_tree::p4est_topidx_t, + quadrant_ptr::Ptr{p4est_quadrant_t}) + # Extract a reference to the tree which_tree + forest = forest_ptr[] + tree = p4est_tree_array_index(forest.trees, which_tree)[] + quadrant = quadrant_ptr[] + q = P4est_wrapper.p4est_quadrant_array_index(tree.quadrants, current_quadrant_index_within_tree) + @assert p4est_quadrant_compare(q, quadrant_ptr) == 0 + user_data = unsafe_wrap(Array, + Ptr{Cint}(forest.user_pointer), + current_quadrant_index_among_trees+1)[current_quadrant_index_among_trees+1] + unsafe_store!(Ptr{Cint}(quadrant.p.user_data), user_data, 1) + current_quadrant_index_within_tree = (current_quadrant_index_within_tree + 1) % (tree.quadrants.elem_count) + current_quadrant_index_among_trees = current_quadrant_index_among_trees+1 + return nothing + end + @cfunction($init_fn_callback, + Cvoid, (Ptr{p4est_t}, p4est_topidx_t, Ptr{p4est_quadrant_t})) +end + +function pXest_reset_callbacks(::P8estType) + # Variables which are updated accross calls to init_fn_callback + current_quadrant_index_within_tree = Cint(0) + current_quadrant_index_among_trees = Cint(0) + + function init_fn_callback(forest_ptr::Ptr{p8est_t}, + which_tree::p4est_topidx_t, + quadrant_ptr::Ptr{p8est_quadrant_t}) + # Extract a reference to the tree which_tree + forest = forest_ptr[] + tree = p8est_tree_array_index(forest.trees, which_tree)[] + quadrant = quadrant_ptr[] + q = P4est_wrapper.p8est_quadrant_array_index(tree.quadrants, current_quadrant_index_within_tree) + @assert p8est_quadrant_compare(q, quadrant_ptr) == 0 + user_data = unsafe_wrap(Array, + Ptr{Cint}(forest.user_pointer), + current_quadrant_index_among_trees+1)[current_quadrant_index_among_trees+1] + unsafe_store!(Ptr{Cint}(quadrant.p.user_data), user_data, 1) + current_quadrant_index_within_tree = (current_quadrant_index_within_tree + 1) % (tree.quadrants.elem_count) + current_quadrant_index_among_trees = current_quadrant_index_among_trees+1 + return nothing + end + init_fn_callback_c = @cfunction($init_fn_callback, + Cvoid, (Ptr{p8est_t}, p4est_topidx_t, Ptr{p8est_quadrant_t})) +end + + +function p2est_quadrant_is_equal(a,b) + a[].z==b[].z && a[].level==b[].level +end + +function P4EST_QUADRANT_LEN(l) + p4est_qcoord_t(1) << (P4est_wrapper.P4EST_MAXLEVEL-l) +end + +function p2est_quadrant_is_ancestor(a,b) + if (a[].level==b[].level) + return false + end + return (b[].z >= a[].z && + b[].z < a[].z + P4EST_QUADRANT_LEN(a[].level)) +end + +function p6est_vertically_adapt_reset_callbacks() + current_quadrant_index_among_trees = Cint(-1) + current_quadrant_index_within_tree = Cint(0) + current_layer_within_column = Cint(0) + current_layer = Cint(0) + previous_quadrant = Ref{p4est_quadrant_t}() + + + function init_fn_callback(forest_ptr::Ptr{p6est_t}, + which_tree::p4est_topidx_t, + column_ptr::Ptr{p4est_quadrant_t}, + layer_ptr::Ptr{p2est_quadrant_t}) + + forest = forest_ptr[] + columns = forest.columns[] + tree = p4est_tree_array_index(columns.trees, which_tree)[] + quadrant = column_ptr[] + layer = layer_ptr[] + + if (current_quadrant_index_among_trees==-1 || + p4est_quadrant_compare(previous_quadrant,column_ptr) != 0) + + previous_quadrant = column_ptr + current_quadrant_index_among_trees = current_quadrant_index_among_trees+1 + current_quadrant_index_within_tree = (current_quadrant_index_within_tree + 1) % (tree.quadrants.elem_count) + q = p4est_quadrant_array_index(tree.quadrants, current_quadrant_index_within_tree) + #@assert p4est_quadrant_compare(q,column_ptr) == 0 + current_layer_within_column = 0 + end + + q = p4est_quadrant_array_index(tree.quadrants, current_quadrant_index_within_tree) + #@assert p4est_quadrant_compare(q,column_ptr) == 0 + + + user_data = unsafe_wrap(Array, Ptr{Cint}(forest.user_pointer), current_layer+1)[current_layer+1] + + + f,l=P6EST_COLUMN_GET_RANGE(column_ptr[]) + q2_ptr=p2est_quadrant_array_index(forest.layers[], f+current_layer_within_column) + @assert p2est_quadrant_is_equal(q2_ptr,layer_ptr) + + unsafe_store!(Ptr{Cint}(q2_ptr[].p.user_data), user_data, 1) + + + current_layer_within_column=current_layer_within_column+1 + current_layer=current_layer+1 + + if (which_tree==columns.connectivity[].num_trees && + current_quadrant_index_within_tree+1==tree.quadrants.elem_count && + current_layer_within_column==l-f) + current_quadrant_index_among_trees = Cint(-1) + current_layer=Cint(0) + end + + return nothing + end + + @cfunction($init_fn_callback, Cvoid, + (Ptr{p6est_t}, p4est_topidx_t, Ptr{p4est_quadrant_t},Ptr{p2est_quadrant_t})) + +end + +function p6est_vertically_refine_callbacks() + function refine_layer_callback(p6est::Ptr{p6est_t}, + which_tree::p4est_topidx_t, + column::Ptr{p4est_quadrant_t}, + layer::Ptr{p2est_quadrant_t}) + Cint(unsafe_wrap(Array, Ptr{Cint}(layer[].p.user_data), 1)[] == refine_flag) + end + + refine_layer_fn_callback_c = @cfunction($refine_layer_callback, Cint, + (Ptr{p6est_t}, p4est_topidx_t, Ptr{p4est_quadrant_t}, Ptr{p2est_quadrant_t})) + + function refine_layer_replace_callback(p6est::Ptr{p6est_t}, + which_tree::p4est_topidx_t, + num_outcolumns::Cint, + num_outlayers::Cint, + outcolumns::Ptr{Ptr{p4est_quadrant_t}}, + outlayers::Ptr{Ptr{p2est_quadrant_t}}, + num_incolumns::Cint, + num_inlayers::Cint, + incolumns::Ptr{Ptr{p4est_quadrant_t}}, + inlayers::Ptr{Ptr{p2est_quadrant_t}}) + + @assert num_outcolumns==1 + @assert num_outlayers==1 + @assert num_incolumns==1 + @assert num_inlayers==2 + + inlayers_array=unsafe_wrap(Array, inlayers, num_inlayers) + for i=1:num_inlayers + quadrant = inlayers_array[i][] + if (quadrant.p.user_data) != C_NULL + unsafe_store!(Ptr{Cint}(quadrant.p.user_data), nothing_flag, 1) + end + end + end + + refine_layer_replace_callback_c = @cfunction($refine_layer_replace_callback, Cvoid, + (Ptr{p6est_t}, p4est_topidx_t, Cint, Cint, + Ptr{Ptr{p4est_quadrant_t}}, Ptr{Ptr{p2est_quadrant_t}}, Cint, Cint, + Ptr{Ptr{p4est_quadrant_t}}, Ptr{Ptr{p2est_quadrant_t}})) + + refine_layer_fn_callback_c, refine_layer_replace_callback_c +end + + +function pXest_coarsen_callbacks(::P4estType) + function coarsen_callback(forest_ptr::Ptr{p4est_t}, + which_tree::p4est_topidx_t, + quadrant_ptr::Ptr{Ptr{p4est_quadrant_t}}) + + num_children=get_num_children(Val{2}) + quadrants=unsafe_wrap(Array, quadrant_ptr, num_children) + coarsen=Cint(1) + for quadrant_index=1:num_children + quadrant = quadrants[quadrant_index][] + # I have noticed that new quadrants created as by-product + # of the refininement process have quadrant.p.user_data == C_NULL + # Not sure why ... The following if-end takes care of this. + if (quadrant.p.user_data) == C_NULL + return Cint(0) + end + is_coarsen_flag=(unsafe_wrap(Array,Ptr{Cint}(quadrant.p.user_data),1)[])==coarsen_flag + if (!is_coarsen_flag) + return Cint(0) + end + end + return coarsen + end + coarsen_fn_callback_c = @cfunction($coarsen_callback, + Cint, (Ptr{p4est_t}, p4est_topidx_t, Ptr{Ptr{p4est_quadrant_t}})) +end + +function pXest_coarsen_callbacks(::P8estType) + function coarsen_callback(forest_ptr::Ptr{p8est_t}, + which_tree::p4est_topidx_t, + quadrant_ptr::Ptr{Ptr{p8est_quadrant_t}}) + + num_children=get_num_children(Val{3}) + quadrants=unsafe_wrap(Array, quadrant_ptr, num_children) + coarsen=Cint(1) + for quadrant_index=1:num_children + quadrant = quadrants[quadrant_index][] + # I have noticed that new quadrants created as by-product + # of the refininement process have quadrant.p.user_data == C_NULL + # Not sure why ... The following if-end takes care of this. + if (quadrant.p.user_data) == C_NULL + return Cint(0) + end + is_coarsen_flag=(unsafe_wrap(Array,Ptr{Cint}(quadrant.p.user_data),1)[])==coarsen_flag + if (!is_coarsen_flag) + return Cint(0) + end + end + return coarsen + end + coarsen_fn_callback_c = @cfunction($coarsen_callback, + Cint, (Ptr{p8est_t}, p4est_topidx_t, Ptr{Ptr{p8est_quadrant_t}})) +end + + +function pXest_refine_callbacks(::P4estType) + function refine_callback(::Ptr{p4est_t}, + which_tree::p4est_topidx_t, + quadrant_ptr::Ptr{p4est_quadrant_t}) + quadrant = quadrant_ptr[] + return Cint(unsafe_wrap(Array, Ptr{Cint}(quadrant.p.user_data), 1)[] == refine_flag) + end + refine_callback_c = @cfunction($refine_callback, Cint, (Ptr{p4est_t}, p4est_topidx_t, Ptr{p4est_quadrant_t})) + + function refine_replace_callback(::Ptr{p4est_t}, + which_tree::p4est_topidx_t, + num_outgoing::Cint, + outgoing_ptr::Ptr{Ptr{p4est_quadrant_t}}, + num_incoming::Cint, + incoming_ptr::Ptr{Ptr{p4est_quadrant_t}}) + num_children=get_num_children(Val{2}) + @assert num_outgoing==1 + @assert num_incoming==num_children + outgoing=unsafe_wrap(Array, outgoing_ptr, 1) + quadrant = outgoing[1][] + incoming=unsafe_wrap(Array, incoming_ptr, num_children) + for quadrant_index=1:num_children + quadrant = incoming[quadrant_index][] + if (quadrant.p.user_data) != C_NULL + unsafe_store!(Ptr{Cint}(quadrant.p.user_data), nothing_flag, 1) + end + end + end + refine_replace_callback_c = + @cfunction($refine_replace_callback, Cvoid, (Ptr{p4est_t}, + p4est_topidx_t, + Cint, + Ptr{Ptr{p4est_quadrant_t}}, + Cint, + Ptr{Ptr{p4est_quadrant_t}})) + refine_callback_c, refine_replace_callback_c +end + +function pXest_refine_callbacks(::P8estType) + function refine_callback(::Ptr{p8est_t}, + which_tree::p4est_topidx_t, + quadrant_ptr::Ptr{p8est_quadrant_t}) + quadrant = quadrant_ptr[] + return Cint(unsafe_wrap(Array, Ptr{Cint}(quadrant.p.user_data), 1)[] == refine_flag) + end + refine_callback_c = @cfunction($refine_callback, Cint, (Ptr{p8est_t}, p4est_topidx_t, Ptr{p8est_quadrant_t})) + + function refine_replace_callback(::Ptr{p8est_t}, + which_tree::p4est_topidx_t, + num_outgoing::Cint, + outgoing_ptr::Ptr{Ptr{p8est_quadrant_t}}, + num_incoming::Cint, + incoming_ptr::Ptr{Ptr{p8est_quadrant_t}}) + num_children=get_num_children(Val{3}) + @assert num_outgoing==1 + @assert num_incoming==num_children + outgoing=unsafe_wrap(Array, outgoing_ptr, 1) + quadrant = outgoing[1][] + incoming=unsafe_wrap(Array, incoming_ptr, num_children) + for quadrant_index=1:num_children + quadrant = incoming[quadrant_index][] + if (quadrant.p.user_data) != C_NULL + unsafe_store!(Ptr{Cint}(quadrant.p.user_data), nothing_flag, 1) + end + end + end + + refine_replace_callback_c = + @cfunction($refine_replace_callback, Cvoid, (Ptr{p8est_t}, + p4est_topidx_t, + Cint, + Ptr{Ptr{p8est_quadrant_t}}, + Cint, + Ptr{Ptr{p8est_quadrant_t}})) + + refine_callback_c, refine_replace_callback_c +end + +function _unwrap_ghost_quadrants(::P4estType, pXest_ghost) + Ptr{p4est_quadrant_t}(pXest_ghost.ghosts.array) +end + +function _unwrap_ghost_quadrants(::P6estType, pXest_ghost) + Ptr{p2est_quadrant_t}(pXest_ghost.ghosts.array) +end + +function _unwrap_ghost_quadrants(::P8estType, pXest_ghost) + Ptr{p8est_quadrant_t}(pXest_ghost.ghosts.array) +end + +function _unwrap_global_first_quadrant(::P4P8estType, pXest) + unsafe_wrap(Array, + pXest.global_first_quadrant, + pXest.mpisize+1) +end + +function _unwrap_global_first_quadrant(::P6estType, pXest) + unsafe_wrap(Array, + pXest.global_first_layer, + pXest.mpisize+1) +end + + +function setup_cell_prange(pXest_type::PXestType, + parts::AbstractVector{<:Integer}, + ptr_pXest, + ptr_pXest_ghost) + comm = parts.comm + + pXest_ghost = ptr_pXest_ghost[] + pXest = ptr_pXest[] + + # Obtain ghost quadrants + ptr_ghost_quadrants = _unwrap_ghost_quadrants(pXest_type,pXest_ghost) + proc_offsets = unsafe_wrap(Array, pXest_ghost.proc_offsets, pXest_ghost.mpisize+1) + + global_first_quadrant = _unwrap_global_first_quadrant(pXest_type,pXest) + + noids,firstgid,gho_to_glo,gho_to_own=map(parts) do part + gho_to_glo = Vector{Int}(undef, pXest_ghost.ghosts.elem_count) + gho_to_own = Vector{Int32}(undef, pXest_ghost.ghosts.elem_count) + k=1 + for i=1:pXest_ghost.mpisize + for j=proc_offsets[i]:proc_offsets[i+1]-1 + quadrant = ptr_ghost_quadrants[j+1] + piggy3 = quadrant.p.piggy3 + gho_to_glo[k] = global_first_quadrant[i]+piggy3.local_num+1 + gho_to_own[k] = Int32(i) + k=k+1 + end + end + global_first_quadrant[part+1]-global_first_quadrant[part],global_first_quadrant[part]+1,gho_to_glo,gho_to_own + end |> tuple_of_arrays + ngids = global_first_quadrant[end]+1 + + partition=map(parts,noids,firstgid,gho_to_glo,gho_to_own) do part, noids, firstgid, gho_to_glo, gho_to_own + owner = part + own_indices=OwnIndices(ngids,owner,(collect(firstgid:firstgid+noids-1))) + ghost_indices=GhostIndices(ngids,gho_to_glo,gho_to_own) + OwnAndGhostIndices(own_indices,ghost_indices) + end + # This is required to provide the hint that the communication + # pattern underlying partition is symmetric, so that we do not have + # to execute the algorithm the reconstructs the reciprocal in the + # communication graph + assembly_neighbors(partition;symmetric=true) + PRange(partition) +end + + +# To add to P4est_wrapper.jl library +# I just translated this function to Julia from its p4est counterpart +# We cannot call it directly because it is declared as static within p4est, +# and thus it does not belong to the ABI of the dynamic library object. + +# /** Decode the face_code into hanging face information. +# * +# * This is mostly for demonstration purposes. Applications probably will +# * integrate it into their own loop over the face for performance reasons. +# * +# * \param[in] face_code as in the p4est_lnodes_t structure. +# * \param[out] hanging face: if there are hanging faces, +# * hanging_face = -1 if the face is not hanging, +# * = 0 if the face is the first half, +# * = 1 if the face is the second half. +# * note: not touched if there are no hanging faces. +# * \return true if any face is hanging, false otherwise. +# */ + +const p4est_corner_faces = [0 2; 1 2; 0 3; 1 3] +const p4est_corner_face_corners = [0 -1 0 -1; -1 0 1 -1; 1 -1 -1 0; -1 1 -1 1] +function p4est_lnodes_decode(face_code, hanging_face) + @assert face_code >= 0 + if (face_code != 0) + c = face_code & 0x03 + work = face_code >> 2 + hanging_face .= -1 + for i = 0:1 + f = p4est_corner_faces[c+1, i+1] + hanging_face[f+1] = (work & 0x01) != 0 ? p4est_corner_face_corners[c+1, f+1] : -1 + work >>= 1 + end + return 1 + else + return 0 + end +end + +const p8est_corner_faces = [0 2 4; 1 2 4; 0 3 4; 1 3 4; 0 2 5; 1 2 5; 0 3 5; 1 3 5] +const p8est_face_edges = [ 4 6 8 10; 5 7 9 11; 0 2 8 9; 1 3 10 11; 0 1 4 5; 2 3 6 7] +const p8est_corner_face_corners = [0 -1 0 -1 0 -1; -1 0 1 -1 1 -1 ; 1 -1 -1 0 2 -1 ; -1 1 -1 1 3 -1 ; + 2 -1 2 -1 -1 0; -1 2 3 -1 -1 1 ; 3 -1 -1 2 -1 2 ; -1 3 -1 3 -1 3 ] +const p8est_corner_edges = [ 0 4 8; 0 5 9; 1 4 10; 1 5 11; 2 6 8; 2 7 9; 3 6 10; 3 7 11 ] + +# To add to p8est_wrapper.jl library +# I just translated this function to Julia from its p4est counterpart +# We cannot call it directly because it is declared as static within p4est, +# and thus it does not belong to the ABI of the dynamic library object. +function p8est_lnodes_decode(face_code, + hanging_face, + hanging_edge) + @assert face_code >= 0 + if (face_code!=0) + c = face_code & 0x0007 + work = face_code >> 3 + hanging_face .= -1 + hanging_edge .= -1 + cwork = c + for i=0:2 + if ((work & 0x0001)!=0) + f = p8est_corner_faces[c+1,i+1] + hanging_face[f+1] = p8est_corner_face_corners[c+1,f+1] + for j=0:3 + e = p8est_face_edges[f+1,j+1] + hanging_edge[e+1] = 4 + end + end + work >>= 1 + end + for i=0:3 + if ((work & 0x0001)!=0) + e = p8est_corner_edges[c+1,i+1] + hanging_edge[e+1] = (hanging_edge[e+1] == -1) ? 0 : 2 + hanging_edge[e+1] += (cwork & 0x0001) + end + cwork >>= 1 + work >>= 1 + end + return 1 + else + return 0 + end +end + +function p6est_lnodes_decode(face_code, + hanging_face, + hanging_edge) + @assert face_code >= 0 + if (face_code != 0) + fc4 = face_code & 0x000f + h = Int16((face_code & 0x0010) >> 4) + work = Int16(face_code >> 5) + hanging_face .= -1 + hanging_edge .= -1 + p4est_lnodes_decode(fc4, hanging_face[3:end]) + for f=0:3 + hf = hanging_face[f + 3] + w = work & 0x0001 + if (hf >= 0) + hanging_edge[p8est_face_edges[f+3,3]+1] = 2 + hf + hanging_edge[p8est_face_edges[f+3,4]+1] = 2 + hf + hanging_edge[p8est_face_edges[f+3,1⊻hf+1]+1] = 4 + if (w!=0) + hanging_edge[p8est_face_edges[f+3,3⊻h+1]+1] = 4 + hanging_edge[p8est_face_edges[f+3,1⊻hf+1]+1] = 4 + hanging_edge[p8est_face_edges[f+3,hf+1]+1] = 2+h + hanging_face[f + 3] = (hf << 1) | h + else + hanging_face[f + 3] = 4 + hf + end + elseif (w!=0) + hanging_edge[p8est_face_edges[f+3,3⊻h+1]+1] = 4; + hanging_edge[p8est_face_edges[f+3,1]+1] = + max(hanging_edge[p8est_face_edges[f+3,1]+1], 2+h); + hanging_edge[p8est_face_edges[f+3,2]+1] = + max(hanging_edge[p8est_face_edges[f+3,2]+1], 2+h); + hanging_face[f + 3] = 6 + h; + end + work >>= 1; + end + + for e=0:3 + if ((work & 0x0001)!=0) + if (hanging_edge[e+1] < 0) + hanging_edge[e+1] = h + else + @assert (hanging_edge[e+1] == 2 + h || hanging_edge[e+1] == 4) + end + end + work >>= 1; + end + return 1 + else + return 0 + end +end + +function pXest_lnodes_decode(::P6estType,face_code, hanging_face, hanging_edge) + p6est_lnodes_decode(face_code, hanging_face, hanging_edge) +end + +function pXest_lnodes_decode(::P8estType,face_code, hanging_face, hanging_edge) + p8est_lnodes_decode(face_code, hanging_face, hanging_edge) +end + +function num_cell_dims(::P4estType) + 2 +end + +function num_cell_dims(::P6P8estType) + 3 +end + +function _faces_to_cell_element_nodes(pXest_type::P4P8estType) + Dc = num_cell_dims(pXest_type) + nf = num_cell_faces(Val{Dc}) + collect(1:nf) +end + +function _faces_to_cell_element_nodes(pXest_type::P6estType) + [13,15, 11,17, 5,23] +end + +function _edges_to_cell_element_nodes(pXest_type::P8estType) + Dc = num_cell_dims(pXest_type) + nf = num_cell_faces(Val{Dc}) + ne = num_cell_edges(Val{Dc}) + collect(nf+1:nf+ne) +end + +function _edges_to_cell_element_nodes(pXest_type::P6estType) + [2,8,20,26, 4,6,22,24, 10,12,16,18] +end + +function _vertices_to_cell_element_nodes(pXest_type::P4P8estType) + Dc = num_cell_dims(pXest_type) + nf = num_cell_faces(Val{Dc}) + ne = num_cell_edges(Val{Dc}) + nv = num_cell_vertices(Val{Dc}) + collect(nf+ne+1:nf+ne+nv) +end + +function _vertices_to_cell_element_nodes(pXest_type::P6estType) + [1,3,7,9, 19,21,25,27] +end + +function _build_map_from_faces_to_cell_lface(vnodes, element_nodes, face_code) + n_cell_faces = num_cell_faces(Val{2}) + hanging_face = Vector{Cint}(undef, n_cell_faces) + + # Build a map from faces to (cell,lface) + p4est_gface_to_gcell_p4est_lface = Dict{Int,Tuple{Int,Int}}() + for cell = 1:length(face_code) + start = (cell - 1) * vnodes + 1 + p4est_cell_faces = view(element_nodes, start:start+n_cell_faces-1) + has_hanging = p4est_lnodes_decode(face_code[cell], hanging_face) + if (has_hanging==0) + for (lface, gface) in enumerate(p4est_cell_faces) + p4est_gface_to_gcell_p4est_lface[gface] = (cell, lface) + end + else + for (lface, half) in enumerate(hanging_face) + # Current face is NOT hanging + if (half == -1) + gface = p4est_cell_faces[lface] + p4est_gface_to_gcell_p4est_lface[gface] = (cell, lface) + end + end + end + end + p4est_gface_to_gcell_p4est_lface +end + + +function _build_map_from_faces_edges_to_cell_lface_ledge(pXest_type, vnodes, element_nodes, face_code) + + faces_to_cell_element_nodes = _faces_to_cell_element_nodes(pXest_type) + edges_to_cell_element_nodes = _edges_to_cell_element_nodes(pXest_type) + + + n_cell_faces = num_cell_faces(Val{3}) + n_cell_edges = num_cell_edges(Val{3}) + + hanging_face = Vector{Cint}(undef, n_cell_faces) + hanging_edge = Vector{Cint}(undef, n_cell_edges) + + # Build a map from faces to (cell,lface) + p4est_gface_to_gcell_p4est_lface = Dict{Int,Tuple{Int,Int}}() + p4est_gedge_to_gcell_p4est_ledge = Dict{Int,Tuple{Int,Int}}() + for cell = 1:length(face_code) + s = (cell - 1) * vnodes + 1 + e = cell * vnodes + p4est_cell_nodes = view(element_nodes, s:e) + p4est_cell_faces = view(p4est_cell_nodes,faces_to_cell_element_nodes) + p4est_cell_edges = view(p4est_cell_nodes,edges_to_cell_element_nodes) + + + has_hanging = pXest_lnodes_decode(pXest_type, face_code[cell], hanging_face, hanging_edge) + if (has_hanging==0) + for (lface, gface) in enumerate(p4est_cell_faces) + p4est_gface_to_gcell_p4est_lface[gface] = (cell, lface) + end + for (ledge, gedge) in enumerate(p4est_cell_edges) + p4est_gedge_to_gcell_p4est_ledge[gedge] = (cell, ledge) + end + else + for (lface, half) in enumerate(hanging_face) + # Current face is NOT hanging + if (half == -1) + gface = p4est_cell_faces[lface] + p4est_gface_to_gcell_p4est_lface[gface] = (cell, lface) + end + end + for (ledge, half) in enumerate(hanging_edge) + # Current edge is NOT hanging + if (half == -1) + gedge = p4est_cell_edges[ledge] + p4est_gedge_to_gcell_p4est_ledge[gedge] = (cell, ledge) + end + end + end + end + p4est_gface_to_gcell_p4est_lface, p4est_gedge_to_gcell_p4est_ledge +end + +function pXest_2_gridap_vertex(::Type{Val{Dc}}) where Dc + Gridap.Arrays.IdentityVector(num_cell_vertices(Val{Dc})) +end + +function p8est_2_gridap_edge() + Gridap.Arrays.IdentityVector(num_cell_edges(Val{3})) +end + +function pXest_2_gridap_facet(::Type{Val{Dc}}) where Dc + if (Dc==2) + GridapP4est.P4EST_2_GRIDAP_FACET_2D + else + @assert Dc==3 + GridapP4est.P4EST_2_GRIDAP_FACET_3D + end +end + +function hanging_lvertex_within_face_2d(half) + half == 0 ? 1 : 0 +end + +function hanging_lvertex_within_face_3d(half) + if (half==0) + return 3 + elseif (half==1) + return 2 + elseif (half==2) + return 1 + elseif (half==3) + return 0 + end +end + +function hanging_lvertex_within_edge(half) + if (half==0 || half==2) + return 1 + elseif (half==1 || half==3) + return 0 + end + @assert false +end + +function regular_lvertex_within_face(half) + return half +end + +function regular_lvertex_within_edge(half) + if (half==0 || half==2) + return 0 + elseif (half==1 || half==3) + return 1 + end + @assert false +end + + +function subface_to_hanging_edges_within_subface(::P6estType) + p6est_subface_to_hanging_edges_within_subface +end + +function subface_to_hanging_edges_within_subface(::P8estType) + p8est_subface_to_hanging_edges_within_subface +end + +function subface_to_hanging_edges_within_face(::P6estType) + p6est_subface_to_hanging_edges_within_face +end + +function subface_to_hanging_edges_within_face(::P8estType) + p8est_subface_to_hanging_edges_within_face +end + +function generate_cell_faces_and_non_conforming_glue(pXest_type::PXestType, + ptr_pXest_lnodes, + cell_prange) + + Dc = num_cell_dims(pXest_type) + n_cell_vertices = num_cell_vertices(Val{Dc}) + n_cell_edges = num_cell_edges(Val{Dc}) + n_cell_faces = num_cell_faces(Val{Dc}) + + lnodes = ptr_pXest_lnodes[] + element_nodes = unsafe_wrap(Array, lnodes.element_nodes, lnodes.vnodes * lnodes.num_local_elements) + face_code = unsafe_wrap(Array, lnodes.face_code, lnodes.num_local_elements) + println(face_code) + hanging_face = Vector{Cint}(undef, n_cell_faces) + face_code_with_ghosts = map(partition(cell_prange)) do indices + @assert length(face_code)==own_length(indices) + @assert own_length(indices)==lnodes.num_local_elements + face_code_with_ghosts=similar(face_code, local_length(indices)) + face_code_with_ghosts[1:own_length(indices)] .= face_code + face_code_with_ghosts + end + + cache_face_code=fetch_vector_ghost_values_cache(face_code_with_ghosts, partition(cell_prange)) + fetch_vector_ghost_values!(face_code_with_ghosts, cache_face_code) |> wait + + element_nodes_with_ghosts = map(partition(cell_prange)) do indices + nonlocal_nodes = unsafe_wrap(Array, lnodes.nonlocal_nodes, lnodes.num_local_nodes-lnodes.owned_count) + element_nodes_with_ghosts_data=similar(element_nodes, local_length(indices)*lnodes.vnodes) + for (i,node) in enumerate(element_nodes) + if (node wait + + map(element_nodes_with_ghosts,face_code_with_ghosts,partition(cell_prange)) do element_nodes_with_ghosts, face_code_with_ghosts, indices + @debug "ENDES_WO_GHOSTS[$(part_id(indices))]: $(element_nodes)" + @debug "ENDES_WITH_GHOSTS[$(part_id(indices))]: $(element_nodes_with_ghosts.data)" + @debug "FCODS_WO_GHOSTS[$(part_id(indices))]: $(face_code)" + @debug "FCODS_WITH_GHOSTS[$(part_id(indices))]: $(face_code_with_ghosts)" + end + + if (Dc==2) + hanging_lvertex_within_face=hanging_lvertex_within_face_2d + pXest_face_corners = p4est_face_corners + else + hanging_lvertex_within_face=hanging_lvertex_within_face_3d + pXest_face_corners = p8est_face_corners + end + + if (Dc==3) + hanging_edge = Vector{Cint}(undef, n_cell_edges) + end + + num_regular_faces, + num_hanging_faces, + gridap_cell_faces, + hanging_faces_glue = + map(partition(cell_prange), + element_nodes_with_ghosts, + face_code_with_ghosts) do indices, + element_nodes_with_ghosts, + face_code_with_ghosts + @assert local_length(indices)==length(face_code_with_ghosts) + + num_local_elements = local_length(indices) + num_regular_faces = Vector{Int}(undef, Dc) + num_hanging_faces = Vector{Int}(undef, Dc) + + # Count regular vertices + num_regular_faces[1] = 0 + regular_vertices_p4est_to_gridap = Dict{Int,Int}() + + num_regular_faces[Dc] = 0 + regular_faces_p4est_to_gridap = Dict{Int,Int}() + + if (Dc==2) + p4est_gface_to_gcell_p4est_lface = + _build_map_from_faces_to_cell_lface(lnodes.vnodes, element_nodes_with_ghosts.data, face_code_with_ghosts) + else + p4est_gface_to_gcell_p4est_lface, + p4est_gedge_to_gcell_p4est_ledge = + _build_map_from_faces_edges_to_cell_lface_ledge(pXest_type, + lnodes.vnodes, + element_nodes_with_ghosts.data, + face_code_with_ghosts) + end + + PXEST_2_GRIDAP_VERTEX = pXest_2_gridap_vertex(Val{Dc}) + PXEST_2_GRIDAP_FACE = pXest_2_gridap_facet(Val{Dc}) + PXEST_2_GRIDAP_EDGE = p8est_2_gridap_edge() + + n = local_length(indices) + gridap_cell_vertices_ptrs = Vector{Int32}(undef,n+1) + gridap_cell_faces_ptrs = Vector{Int32}(undef,n+1) + gridap_cell_vertices_ptrs[1]=1 + gridap_cell_faces_ptrs[1]=1 + + hanging_vertices_pairs_to_owner_face = Dict{Tuple{Int,Int},Int}() + hanging_faces_pairs_to_owner_face = Dict{Tuple{Int,Int},Tuple{Int,Int}}() + for i=1:n + gridap_cell_vertices_ptrs[i+1]=gridap_cell_vertices_ptrs[i]+n_cell_vertices + gridap_cell_faces_ptrs[i+1]=gridap_cell_faces_ptrs[i]+n_cell_faces + end + + gridap_cell_vertices_data = Vector{Int}(undef, num_local_elements * n_cell_vertices) + gridap_cell_vertices_data .= -1 + + gridap_cell_faces_data = Vector{Int}(undef, num_local_elements * n_cell_faces) + gridap_cell_faces_data .= -1 + + if (Dc==3) + num_regular_faces[2] = 0 + gridap_cell_edges_ptrs = Vector{Int32}(undef,n+1) + gridap_cell_edges_ptrs[1]=1 + for i=1:n + gridap_cell_edges_ptrs[i+1]=gridap_cell_edges_ptrs[i]+n_cell_edges + end + gridap_cell_edges_data = Vector{Int}(undef, num_local_elements * n_cell_edges) + gridap_cell_edges_data .= -1 + hanging_edges_cell_ledge_to_owner_face_half = Dict{Tuple{Int,Int},Tuple{Int,Int}}() + owner_edge_subedge_to_cell_ledge = Dict{Tuple{Int,Int},Tuple{Int,Int}}() + hanging_vertices_pairs_to_owner_edge = Dict{Tuple{Int,Int},Int}() + regular_edges_p4est_to_gridap = Dict{Int,Int}() + end + + faces_to_cell_element_nodes = _faces_to_cell_element_nodes(pXest_type) + if (Dc==3) + edges_to_cell_element_nodes = _edges_to_cell_element_nodes(pXest_type) + end + + vertices_to_cell_element_nodes = _vertices_to_cell_element_nodes(pXest_type) + + for cell = 1:num_local_elements + start_gridap_vertices = (cell - 1) * n_cell_vertices + start_gridap_faces = (cell - 1) * n_cell_faces + + s = (cell-1)*lnodes.vnodes + 1 + e = cell*lnodes.vnodes + p4est_cell_nodes = view(element_nodes_with_ghosts.data, s:e) + + p4est_cell_faces = view(p4est_cell_nodes,faces_to_cell_element_nodes) + p4est_cell_vertices = view(p4est_cell_nodes,vertices_to_cell_element_nodes) + + gridap_cell_vertices = view(gridap_cell_vertices_data, + start_gridap_vertices+1:start_gridap_vertices+n_cell_vertices) + gridap_cell_faces = view(gridap_cell_faces_data, + start_gridap_faces+1:start_gridap_faces+n_cell_faces) + + if (Dc==2) + has_hanging = p4est_lnodes_decode(face_code_with_ghosts[cell], hanging_face) + else + has_hanging = pXest_lnodes_decode(pXest_type, face_code_with_ghosts[cell], hanging_face, hanging_edge) + start_gridap_edges = (cell-1)*n_cell_edges + gridap_cell_edges = view(gridap_cell_edges_data, start_gridap_edges+1:start_gridap_edges+n_cell_edges) + p4est_cell_edges = view(p4est_cell_nodes,edges_to_cell_element_nodes) + end + if has_hanging == 0 + # All vertices/edges/faces of the current cell are regular + # Process vertices + for (p4est_lvertex, p4est_gvertex) in enumerate(p4est_cell_vertices) + num_regular_faces[1] = + process_current_face!(gridap_cell_vertices, + regular_vertices_p4est_to_gridap, + num_regular_faces[1], + p4est_cell_vertices, + p4est_lvertex, + p4est_gvertex, + PXEST_2_GRIDAP_VERTEX) + end + + if (Dc==3) + for (p4est_ledge, p4est_gedge) in enumerate(p4est_cell_edges) + num_regular_faces[2] = + process_current_face!(gridap_cell_edges, + regular_edges_p4est_to_gridap, + num_regular_faces[2], + p4est_cell_edges, + p4est_ledge, + p4est_gedge, + PXEST_2_GRIDAP_EDGE) + end + end + + # Process faces + for (p4est_lface, p4est_gface) in enumerate(p4est_cell_faces) + num_regular_faces[Dc] = + process_current_face!(gridap_cell_faces, + regular_faces_p4est_to_gridap, + num_regular_faces[Dc], + p4est_cell_faces, + p4est_lface, + p4est_gface, + PXEST_2_GRIDAP_FACE) + end + else + if (isa(pXest_type,P4P8estType)) + # "Touch" hanging vertices before processing current cell + # This is required as we dont have any means to detect + # a hanging vertex from a non-hanging face + for (p4est_lface, half) in enumerate(hanging_face) + if (half != -1) + hanging_vertex_lvertex_within_face = hanging_lvertex_within_face(half) + p4est_lvertex = pXest_face_corners[p4est_lface, + hanging_vertex_lvertex_within_face+1] + gridap_cell_vertices[PXEST_2_GRIDAP_VERTEX[p4est_lvertex+1]] = hanging_vertex_code + end + end + end + + if (Dc==3) + for (p4est_ledge, half) in enumerate(hanging_edge) + if (half != -1 && half !=4) + hanging_vertex_lvertex_within_edge = hanging_lvertex_within_edge(half) + p4est_lvertex = p8est_edge_corners[p4est_ledge, + hanging_vertex_lvertex_within_edge+1] + gridap_cell_vertices[PXEST_2_GRIDAP_VERTEX[p4est_lvertex+1]] = hanging_vertex_code + end + end + end + + # Current cell has at least one hanging face + for (p4est_lface, half) in enumerate(hanging_face) + # Current face is NOT hanging + if (half == -1) + # Process vertices on the boundary of p4est_lface + for p4est_lvertex in pXest_face_corners[p4est_lface, :] + p4est_gvertex = p4est_cell_vertices[p4est_lvertex+1] + if (gridap_cell_vertices[p4est_lvertex+1] != hanging_vertex_code) + num_regular_faces[1] = + process_current_face!(gridap_cell_vertices, + regular_vertices_p4est_to_gridap, + num_regular_faces[1], + p4est_cell_vertices, + p4est_lvertex + 1, + p4est_gvertex, + PXEST_2_GRIDAP_VERTEX) + end + end + # Process non-hanging face + p4est_gface = p4est_cell_faces[p4est_lface] + num_regular_faces[Dc] = + process_current_face!(gridap_cell_faces, + regular_faces_p4est_to_gridap, + num_regular_faces[Dc], + p4est_cell_faces, + p4est_lface, + p4est_gface, + PXEST_2_GRIDAP_FACE) + else # Current face is hanging + owner_face = p4est_cell_faces[p4est_lface] + + if half in 0:3 + # Identify regular vertex and hanging vertex + # Repeat code above for regular vertex + # Special treatment for hanging vertex + regular_vertex_lvertex_within_face = regular_lvertex_within_face(half) + hanging_vertex_lvertex_within_face = hanging_lvertex_within_face(half) + + # Process regular vertex + p4est_regular_lvertex = pXest_face_corners[p4est_lface, regular_vertex_lvertex_within_face+1] + p4est_gvertex = p4est_cell_vertices[p4est_regular_lvertex+1] + num_regular_faces[1] = + process_current_face!(gridap_cell_vertices, + regular_vertices_p4est_to_gridap, + num_regular_faces[1], + p4est_cell_vertices, + p4est_regular_lvertex + 1, + p4est_gvertex, + PXEST_2_GRIDAP_VERTEX) + + # Process hanging vertex + p4est_hanging_lvertex = pXest_face_corners[p4est_lface, hanging_vertex_lvertex_within_face+1] + hanging_vertices_pairs_to_owner_face[(cell, PXEST_2_GRIDAP_VERTEX[p4est_hanging_lvertex+1])] = owner_face + else + # Anisotropic refinement + @assert half in 4:7 + p6est_half_to_regular_vertices = [ 0 1; 2 3; 0 2; 1 3] + for regular_vertex_lvertex_within_face in p6est_half_to_regular_vertices[mod(half,4)+1,:] + # Process regular vertex + p4est_regular_lvertex = pXest_face_corners[p4est_lface, regular_vertex_lvertex_within_face+1] + p4est_gvertex = p4est_cell_vertices[p4est_regular_lvertex+1] + num_regular_faces[1] = + process_current_face!(gridap_cell_vertices, + regular_vertices_p4est_to_gridap, + num_regular_faces[1], + p4est_cell_vertices, + p4est_regular_lvertex + 1, + p4est_gvertex, + PXEST_2_GRIDAP_VERTEX) + end + end + + # Process hanging face + hanging_faces_pairs_to_owner_face[(cell, PXEST_2_GRIDAP_FACE[p4est_lface])] = (owner_face,half+1) + + if (Dc==3) + _subface_to_hanging_edges_within_subface = subface_to_hanging_edges_within_subface(pXest_type) + _subface_to_hanging_edges_within_face = subface_to_hanging_edges_within_face(pXest_type) + + for (i,ledge_within_face) in enumerate(_subface_to_hanging_edges_within_subface[mod(half,4)+1,:]) + p4est_ledge=p8est_face_edges[p4est_lface,ledge_within_face+1] + + @debug "[$(MPI.Comm_rank(MPI.COMM_WORLD))] cell=$(cell) p4est_lface=$(p4est_lface) half=$(half) index=$(mod(half,4)+1) p4est_ledge=$(p4est_ledge) ledge_within_face=$(ledge_within_face)" + + gridap_ledge = PXEST_2_GRIDAP_EDGE[p4est_ledge+1] + # Identify the two edges which are hanging within the face + hanging_edges_cell_ledge_to_owner_face_half[(cell, gridap_ledge)] = + (owner_face,-_subface_to_hanging_edges_within_face[mod(half,4)+1,i]) + gridap_cell_edges[gridap_ledge] = hanging_edge_from_face_code + end + end + + end + end + + + if (Dc==3) + for (p4est_ledge, half) in enumerate(hanging_edge) + # Current edge is NOT hanging + if (half == -1) + # Process vertices on the boundary of p4est_ledge + for p4est_lvertex in p8est_edge_corners[p4est_ledge, :] + p4est_gvertex = p4est_cell_vertices[p4est_lvertex+1] + if (gridap_cell_vertices[p4est_lvertex+1] != hanging_vertex_code) + num_regular_faces[1] = + process_current_face!(gridap_cell_vertices, + regular_vertices_p4est_to_gridap, + num_regular_faces[1], + p4est_cell_vertices, + p4est_lvertex + 1, + p4est_gvertex, + PXEST_2_GRIDAP_VERTEX) + end + end + # Process non-hanging edge + p4est_gedge = p4est_cell_edges[p4est_ledge] + num_regular_faces[2] = + process_current_face!(gridap_cell_edges, + regular_edges_p4est_to_gridap, + num_regular_faces[2], + p4est_cell_edges, + p4est_ledge, + p4est_gedge, + PXEST_2_GRIDAP_EDGE) + else # Current edge is hanging + @debug "[$(MPI.Comm_rank(MPI.COMM_WORLD))] cell=$(cell) hanging_edge=$(hanging_edge) p4est_ledge=$(p4est_ledge) gridap_cell_edges[PXEST_2_GRIDAP_EDGE[p4est_ledge]]: $(gridap_cell_edges[PXEST_2_GRIDAP_EDGE[p4est_ledge]]) half=$(half)" + + if ( gridap_cell_edges[PXEST_2_GRIDAP_EDGE[p4est_ledge]] != hanging_edge_from_face_code ) + # The present hanging edge cannot be within a coarser face + @assert half != 4 + + # # Identify regular vertex and hanging vertex + # # Repeat code above for regular vertex + # # Special treatment for hanging vertex + regular_vertex_lvertex_within_edge = regular_lvertex_within_edge(half) + hanging_vertex_lvertex_within_edge = hanging_lvertex_within_edge(half) + + # # Process regular vertex + p4est_regular_lvertex = p8est_edge_corners[p4est_ledge, regular_vertex_lvertex_within_edge+1] + p4est_gvertex = p4est_cell_vertices[p4est_regular_lvertex+1] + + num_regular_faces[1] = + process_current_face!(gridap_cell_vertices, + regular_vertices_p4est_to_gridap, + num_regular_faces[1], + p4est_cell_vertices, + p4est_regular_lvertex + 1, + p4est_gvertex, + PXEST_2_GRIDAP_VERTEX) + + # Process hanging vertex + p4est_hanging_lvertex = p8est_edge_corners[p4est_ledge, hanging_vertex_lvertex_within_edge+1] + p4est_owner_edge = p4est_cell_edges[p4est_ledge] + hanging_vertices_pairs_to_owner_edge[(cell, + PXEST_2_GRIDAP_VERTEX[p4est_hanging_lvertex+1])] = p4est_owner_edge + + # Process hanging edge + subedge = regular_vertex_lvertex_within_edge+1 + owner_edge_subedge_pair=(p4est_owner_edge,subedge) + gridap_ledge=PXEST_2_GRIDAP_EDGE[p4est_ledge] + hanging_edges_cell_ledge_to_owner_face_half[(cell, gridap_ledge)] = owner_edge_subedge_pair + if (!haskey(owner_edge_subedge_to_cell_ledge,owner_edge_subedge_pair)) + owner_edge_subedge_to_cell_ledge[owner_edge_subedge_pair] = (cell,gridap_ledge) + end + end + end + end + end + end + end + + function is_ghost(cell) + cell>own_length(indices) + end + + # Go over all touched hanging faces and start + # assigning IDs from the last num_regular_faces ID + # For each hanging face, keep track of (owner_cell,lface) + # Go over all hanging faces + # Detect if the owner face is in a ghost cell. + # If not in a ghost cell or touched + # Else + # The current face becomes a regular face + # end + hanging_faces_owner_cell_and_lface = + Vector{Tuple{Int,Int,Int}}(undef, length(keys(hanging_faces_pairs_to_owner_face))) + num_hanging_faces[Dc] = 0 + for key in keys(hanging_faces_pairs_to_owner_face) + (cell, lface) = key + (owner_p4est_gface, half) = hanging_faces_pairs_to_owner_face[key] + num_hanging_faces[Dc] += 1 + start_gridap_faces = (cell - 1) * n_cell_faces + gridap_cell_faces_data[start_gridap_faces+lface] = num_regular_faces[Dc] + num_hanging_faces[Dc] + if (!(is_ghost(cell)) || haskey(regular_faces_p4est_to_gridap,owner_p4est_gface)) + @assert haskey(regular_faces_p4est_to_gridap,owner_p4est_gface) + (owner_cell, p4est_lface) = p4est_gface_to_gcell_p4est_lface[owner_p4est_gface] + hanging_faces_owner_cell_and_lface[num_hanging_faces[Dc]] = + (owner_cell, n_cell_vertices+n_cell_edges+PXEST_2_GRIDAP_FACE[p4est_lface], half) + else + # Glue info cannot be computed for this hanging face + hanging_faces_owner_cell_and_lface[num_hanging_faces[Dc]] = (-1,-1,-1) + end + end + + @debug "[$(MPI.Comm_rank(MPI.COMM_WORLD))] gridap_cell_faces_data: $(gridap_cell_faces_data)" + + + # Go over all touched hanging vertices and start + # assigning IDs from the last num_regular_vertices ID + # For each hanging vertex, keep track of (owner_cell,lface) + num_hanging_faces[1] = 0 + hanging_vertices_owner_cell_and_lface = Tuple{Int,Int,Int}[] + half=1 + owner_p4est_gface_to_hanging_vertex = Dict{Int,Int}() + for key in keys(hanging_vertices_pairs_to_owner_face) + (cell, lvertex) = key + owner_p4est_gface = hanging_vertices_pairs_to_owner_face[key] + if !(haskey(owner_p4est_gface_to_hanging_vertex, owner_p4est_gface)) + num_hanging_faces[1] += 1 + owner_p4est_gface_to_hanging_vertex[owner_p4est_gface] = num_hanging_faces[1] + if (!is_ghost(cell) || (haskey(regular_faces_p4est_to_gridap,owner_p4est_gface))) + (owner_cell, p4est_lface) = p4est_gface_to_gcell_p4est_lface[owner_p4est_gface] + push!(hanging_vertices_owner_cell_and_lface, + (owner_cell, n_cell_vertices+n_cell_edges+PXEST_2_GRIDAP_FACE[p4est_lface],half)) + else + push!(hanging_vertices_owner_cell_and_lface,(-1, -1,-1)) + end + end + start_gridap_vertices = (cell - 1) * n_cell_vertices + gridap_cell_vertices_data[start_gridap_vertices+lvertex] = num_regular_faces[1] + + owner_p4est_gface_to_hanging_vertex[owner_p4est_gface] + end + + @debug "[$(MPI.Comm_rank(MPI.COMM_WORLD))] gridap_cell_vertices_data: $(gridap_cell_vertices_data)" + + if (Dc==3) + half=1 + owner_p4est_gedge_to_hanging_vertex = Dict{Int,Int}() + for key in keys(hanging_vertices_pairs_to_owner_edge) + (cell, lvertex) = key + owner_p4est_gedge = hanging_vertices_pairs_to_owner_edge[key] + if !(haskey(owner_p4est_gedge_to_hanging_vertex, owner_p4est_gedge)) + num_hanging_faces[1] += 1 + owner_p4est_gedge_to_hanging_vertex[owner_p4est_gedge] = num_hanging_faces[1] + if (!is_ghost(cell) || (haskey(regular_edges_p4est_to_gridap,owner_p4est_gedge))) + (owner_cell, p4est_ledge) = p4est_gedge_to_gcell_p4est_ledge[owner_p4est_gedge] + push!(hanging_vertices_owner_cell_and_lface, + (owner_cell, n_cell_vertices+PXEST_2_GRIDAP_EDGE[p4est_ledge],half)) + else + push!(hanging_vertices_owner_cell_and_lface,(-1, -1,-1)) + end + end + start_gridap_vertices = (cell - 1) * n_cell_vertices + gridap_cell_vertices_data[start_gridap_vertices+lvertex] = num_regular_faces[1] + + owner_p4est_gedge_to_hanging_vertex[owner_p4est_gedge] + end + + # Go over all touched hanging edges and start + # assigning IDs from the last num_regular_edge ID + # For each hanging edge, keep track of (owner_cell,lface/ledge) + hanging_edges_owner_cell_and_lface = Tuple{Int,Int,Int}[] + owner_p4est_gface_half_to_hanging_edge = Dict{Tuple{Int,Int},Int}() + owner_p4est_gedge_subedge_to_hanging_edge = Dict{Tuple{Int,Int},Int}() + num_hanging_faces[2] = 0 + ledge_to_cvertices = Gridap.ReferenceFEs.get_faces(HEX, 1, 0) + # The following loop needs (1) the pairs to be traversed in increased order by cell ID; + # (2) gridap cell vertices to be already completed + for key in sort(collect(keys(hanging_edges_cell_ledge_to_owner_face_half))) + (cell, ledge) = key + (owner_p4est_gface_or_gedge, half) = hanging_edges_cell_ledge_to_owner_face_half[key] + @debug "[$(MPI.Comm_rank(MPI.COMM_WORLD))] own_length=$(own_length(indices)) cell=$(cell) ledge=$(ledge) owner_p4est_gface_or_gedge=$(owner_p4est_gface_or_gedge) half=$(half)" + if (half<0) # hanging edge is within a coarser face + owner_p4est_gface = owner_p4est_gface_or_gedge + if !(haskey(owner_p4est_gface_half_to_hanging_edge, (owner_p4est_gface,half))) + num_hanging_faces[2] += 1 + owner_p4est_gface_half_to_hanging_edge[(owner_p4est_gface,half)] = num_hanging_faces[2] + if (!is_ghost(cell) || (haskey(regular_faces_p4est_to_gridap,owner_p4est_gface))) + (owner_cell, p4est_lface) = p4est_gface_to_gcell_p4est_lface[owner_p4est_gface] + push!(hanging_edges_owner_cell_and_lface, + (owner_cell, n_cell_vertices+n_cell_edges+PXEST_2_GRIDAP_FACE[p4est_lface],half)) + else + push!(hanging_edges_owner_cell_and_lface,(-1, -1, -1)) + end + end + start_gridap_edges = (cell - 1) * n_cell_edges + gridap_cell_edges_data[start_gridap_edges+ledge] = num_regular_faces[2] + + owner_p4est_gface_half_to_hanging_edge[(owner_p4est_gface,half)] + + else # hanging edge is within a coarser edge + @assert half==1 || half==2 + owner_p4est_gedge = owner_p4est_gface_or_gedge + owner_gedge_pair = (owner_p4est_gedge,half) + if (haskey(owner_edge_subedge_to_cell_ledge,owner_gedge_pair)) + (owner_cell, owner_cell_ledge) = owner_edge_subedge_to_cell_ledge[owner_gedge_pair] + if (owner_cell==cell) + @assert owner_cell_ledge == ledge + num_hanging_faces[2] += 1 + owner_p4est_gedge_subedge_to_hanging_edge[(owner_p4est_gedge,half)] = num_hanging_faces[2] + if (!is_ghost(cell) || (haskey(regular_edges_p4est_to_gridap,owner_p4est_gedge))) + (owner_cell, p4est_ledge) = p4est_gedge_to_gcell_p4est_ledge[owner_p4est_gedge] + push!(hanging_edges_owner_cell_and_lface, + (owner_cell, n_cell_vertices+PXEST_2_GRIDAP_EDGE[p4est_ledge],half)) + else + push!(hanging_edges_owner_cell_and_lface,(-1, -1, -1)) + end + start_gridap_edges = (cell - 1) * n_cell_edges + gridap_cell_edges_data[start_gridap_edges+ledge] = num_regular_faces[2] + num_hanging_faces[2] + else + haskey_first_subedge = haskey(owner_p4est_gedge_subedge_to_hanging_edge,(owner_p4est_gedge,1)) + haskey_second_subedge = haskey(owner_p4est_gedge_subedge_to_hanging_edge,(owner_p4est_gedge,2)) + if (!(is_ghost(cell))) + @assert haskey_first_subedge && haskey_second_subedge + else + @assert haskey_first_subedge || haskey_second_subedge + end + if (haskey_first_subedge && haskey_second_subedge) + # The following code is required as we may have edges + # with different orientations at the inter-octree boundaries + match=true + start_gridap_vertices_cell = (cell - 1) * n_cell_vertices + start_gridap_vertices_cell_owner = (owner_cell - 1) * n_cell_vertices + for lvertex_cell in ledge_to_cvertices[ledge] + vertex_cell=gridap_cell_vertices_data[start_gridap_vertices_cell+lvertex_cell] + found=false + # Go over vertices of owner_cell_ledge in owner_cell + for lvertex_owner_cell in ledge_to_cvertices[owner_cell_ledge] + vertex_owner_cell=gridap_cell_vertices_data[start_gridap_vertices_cell_owner+lvertex_owner_cell] + if (vertex_owner_cell==vertex_cell) + found=true + break + end + end + if (!found) + match=false + break + end + end + if (match) + owner_half=half + else + owner_half=half==1 ? 2 : 1 + end + elseif (haskey_first_subedge) + owner_half=1 + elseif (haskey_second_subedge) + owner_half=2 + end + @debug "[$(MPI.Comm_rank(MPI.COMM_WORLD))] cell=$(cell) ledge=$(ledge) owner_p4est_gface_or_gedge=$(owner_p4est_gface_or_gedge) half=$(half) owner_half=$(owner_half)" + start_gridap_edges = (cell - 1) * n_cell_edges + gridap_cell_edges_data[start_gridap_edges+ledge] = + num_regular_faces[2] + owner_p4est_gedge_subedge_to_hanging_edge[(owner_p4est_gedge,owner_half)] + end + end + end + end + @debug "[$(MPI.Comm_rank(MPI.COMM_WORLD))] gridap_cell_edges_data: $(gridap_cell_edges_data)" + end + + + gridap_cell_faces = Vector{JaggedArray}(undef,Dc) + gridap_cell_faces[1] = JaggedArray(gridap_cell_vertices_data,gridap_cell_vertices_ptrs) + if (Dc==3) + gridap_cell_faces[2] = JaggedArray(gridap_cell_edges_data,gridap_cell_edges_ptrs) + end + gridap_cell_faces[Dc] = JaggedArray(gridap_cell_faces_data,gridap_cell_faces_ptrs) + + hanging_faces_glue = Vector{Vector{Tuple}}(undef,Dc) + hanging_faces_glue[1] = hanging_vertices_owner_cell_and_lface + if (Dc==3) + hanging_faces_glue[2] = hanging_edges_owner_cell_and_lface + end + hanging_faces_glue[Dc] = hanging_faces_owner_cell_and_lface + + + return num_regular_faces, + num_hanging_faces, + gridap_cell_faces, + hanging_faces_glue + + end |> tuple_of_arrays + + + gridap_cell_faces_out = Vector{MPIArray}(undef,Dc) + for i=1:Dc + gridap_cell_faces_out[i] = map(gridap_cell_faces) do gridap_cell_faces + gridap_cell_faces[i] + end + end + non_conforming_glue=map(num_regular_faces,num_hanging_faces,hanging_faces_glue) do nrf, nhf, hfg + NonConformingGlue(nrf, nhf, hfg) + end + gridap_cell_faces_out,non_conforming_glue + end + + function pXest_tree_array_index(::P4estType, pXest, i) + p4est_tree_array_index(pXest.trees, i) +end + +function pXest_tree_array_index(::P8estType, pXest, i) + p8est_tree_array_index(pXest.trees, i) +end + +function pXest_tree_array_index(::P6estType, pXest, i) + p4est_tree_array_index(pXest.columns[].trees, i) +end + +function pXest_quadrant_array_index(::P4estType, tree, i) + p4est_quadrant_array_index(tree.quadrants, i) +end + +function pXest_quadrant_array_index(::P6estType, tree, i) + p4est_quadrant_array_index(tree.quadrants, i) +end + +function pXest_quadrant_array_index(::P8estType, tree, i) + p8est_quadrant_array_index(tree.quadrants, i) +end + +function pXest_quadrant_is_parent(::P4estType, q1, q2) + p4est_quadrant_is_parent(q1,q2)!=0 +end + +function pXest_quadrant_is_parent(::P8estType, q1, q2) + p8est_quadrant_is_parent(q1,q2)!=0 +end + +function pXest_quadrant_is_equal(::P4estType, q1, q2) + p4est_quadrant_is_equal(q1, q2)!=0 +end + +function pXest_quadrant_is_equal(::P6estType, q1, q2) + Gridap.Helpers.@notimplemented +end + +function pXest_quadrant_is_equal(::P8estType, q1, q2) + p8est_quadrant_is_equal(q1, q2)!=0 +end + +function pXest_cell_coords(::P4estType, q, l) + (q.x,q.y) +end + +function pXest_cell_coords(::P8estType, q, l) + (q.x,q.y,q.z) +end + +function pXest_cell_coords(::P6estType, q, l) + (q.x,q.y,l.z) +end + +function pXest_num_quadrant_layers(::P4P8estType,q) + 1 +end + +function P6EST_COLUMN_GET_RANGE(q) + f=q.p.piggy3.local_num + l=f+q.p.piggy3.which_tree + (f,l) +end + +function pXest_num_quadrant_layers(::P6estType,q) + f,l=P6EST_COLUMN_GET_RANGE(q) + l-f +end + +function pXest_get_layer(::P4P8estType,q,pXest,i) + q +end + +function p2est_quadrant_array_index(sc_array_object::sc_array_t, it) + @assert sc_array_object.elem_size == sizeof(p2est_quadrant_t) + @assert it in 0:sc_array_object.elem_count + return Ptr{p2est_quadrant_t}(sc_array_object.array + sc_array_object.elem_size*it) +end + +function pXest_get_layer(::P6estType,q,pXest,i) + f,_=P6EST_COLUMN_GET_RANGE(q) + p2est_quadrant_array_index(pXest.layers[], f+i)[] +end + +function pXest_get_quadrant_and_layer_levels(::P4P8estType,q,l) + (q.level,) +end + +function pXest_get_quadrant_and_layer_levels(::P6estType,q,l) + (q.level,l.level) +end + +function pXest_get_quadrant_vertex_coordinates(::P4estType, + connectivity::Ptr{p4est_connectivity_t}, + treeid::p4est_topidx_t, + coords, + levels, + corner::Cint, + pvxy::Ptr{Cdouble}) + x,y=coords + level,=levels + p4est_get_quadrant_vertex_coordinates(connectivity, + treeid, + x, + y, + level, + corner, + pvxy) +end + +function pXest_get_quadrant_vertex_coordinates(::P6estType, + connectivity::Ptr{p6est_connectivity_t}, + treeid::p4est_topidx_t, + coords, + levels, + corner::Cint, + pvxy::Ptr{Cdouble}) + x,y,z=coords + qlevel,zlevel=levels + p6est_get_quadrant_vertex_coordinates(connectivity, + treeid, + x, + y, + z, + qlevel, + zlevel, + corner, + pvxy) +end + +function pXest_get_quadrant_vertex_coordinates(::P8estType, + connectivity::Ptr{p8est_connectivity_t}, + treeid::p4est_topidx_t, + coords, + levels, + corner::Cint, + pvxy::Ptr{Cdouble}) + x,y,z=coords + level,=levels + p8est_get_quadrant_vertex_coordinates(connectivity, + treeid, + x, + y, + z, + level, + corner, + pvxy) +end + +function generate_node_coordinates(pXest_type::PXestType, + cell_vertex_lids, + nlvertices, + ptr_pXest_connectivity, + ptr_pXest, + ptr_pXest_ghost) + + Dc = num_cell_dims(pXest_type) + + PXEST_CORNERS=2^Dc + pXest_ghost = ptr_pXest_ghost[] + pXest = ptr_pXest[] + + # Obtain ghost quadrants + ptr_ghost_quadrants = _unwrap_ghost_quadrants(pXest_type, pXest_ghost) + + tree_offsets = unsafe_wrap(Array, pXest_ghost.tree_offsets, pXest_ghost.num_trees+1) + dnode_coordinates=map(cell_vertex_lids,nlvertices) do cell_vertex_lids, nl + node_coordinates=Vector{Point{Dc,Float64}}(undef,nl) + current=1 + vxy=Vector{Cdouble}(undef,Dc) + pvxy=pointer(vxy,1) + cell_lids=cell_vertex_lids.data + for itree=1:pXest_ghost.num_trees + tree = pXest_tree_array_index(pXest_type, pXest, itree-1)[] + # Loop over quadrants/columns in the current tree + for cell=1:tree.quadrants.elem_count + quadrant=pXest_quadrant_array_index(pXest_type,tree,cell-1)[] + # Loop over layers in the current column + for l=1:pXest_num_quadrant_layers(pXest_type,quadrant) + layer=pXest_get_layer(pXest_type, quadrant, pXest, l-1) + coords=pXest_cell_coords(pXest_type,quadrant,layer) + levels=pXest_get_quadrant_and_layer_levels(pXest_type,quadrant,layer) + + for vertex=1:PXEST_CORNERS + pXest_get_quadrant_vertex_coordinates(pXest_type, + ptr_pXest_connectivity, + p4est_topidx_t(itree-1), + coords, + levels, + Cint(vertex-1), + pvxy) + + + @debug "[$(MPI.Comm_rank(MPI.COMM_WORLD))] quadrant=$(cell) layer=$(l) coords=$(coords) levels=$(levels) pvxy=$(unsafe_wrap(Array, pvxy, 3))" + + node_coordinates[cell_lids[current]]=Point{Dc,Float64}(vxy...) + current=current+1 + end + + end + end + end + + # Go over ghost cells + for i=1:pXest_ghost.num_trees + for j=tree_offsets[i]:tree_offsets[i+1]-1 + quadrant = ptr_ghost_quadrants[j+1] + for vertex=1:PXEST_CORNERS + if (Dc==2) + p4est_get_quadrant_vertex_coordinates(ptr_pXest_connectivity, + p4est_topidx_t(i-1), + quadrant.x, + quadrant.y, + quadrant.level, + Cint(vertex-1), + pvxy) + else + p8est_get_quadrant_vertex_coordinates(ptr_pXest_connectivity, + p4est_topidx_t(i-1), + quadrant.x, + quadrant.y, + quadrant.z, + quadrant.level, + Cint(vertex-1), + pvxy) + + end + node_coordinates[cell_lids[current]]=Point{Dc,Float64}(vxy...) + current=current+1 + end + end + end + node_coordinates + end +end + +function generate_grid_and_topology(pXest_type::PXestType, + cell_vertex_lids, + nlvertices, + node_coordinates) + Dc=num_cell_dims(pXest_type) + grid,topology= + map(cell_vertex_lids,nlvertices,node_coordinates) do cell_vertex_lids, nl, node_coordinates + polytope= Dc==2 ? QUAD : HEX + scalar_reffe=Gridap.ReferenceFEs.ReferenceFE(polytope,Gridap.ReferenceFEs.lagrangian,Float64,1) + cell_types=collect(Fill(1,length(cell_vertex_lids))) + cell_reffes=[scalar_reffe] + cell_vertex_lids_gridap=Gridap.Arrays.Table(cell_vertex_lids.data,cell_vertex_lids.ptrs) + grid = Gridap.Geometry.UnstructuredGrid(node_coordinates, + cell_vertex_lids_gridap, + cell_reffes, + cell_types, + Gridap.Geometry.NonOriented()) + + topology = Gridap.Geometry.UnstructuredGridTopology(node_coordinates, + cell_vertex_lids_gridap, + cell_types, + map(Gridap.ReferenceFEs.get_polytope, cell_reffes), + Gridap.Geometry.NonOriented()) + grid,topology + end |> tuple_of_arrays + grid,topology +end + + + + diff --git a/src/UniformlyRefinedForestOfOctreesDiscreteModels.jl b/src/UniformlyRefinedForestOfOctreesDiscreteModels.jl index dce8aac..e66789a 100644 --- a/src/UniformlyRefinedForestOfOctreesDiscreteModels.jl +++ b/src/UniformlyRefinedForestOfOctreesDiscreteModels.jl @@ -1,20 +1,4 @@ -function pXest_lnodes_destroy(::Type{Val{Dc}}, ptr_pXest_lnodes) where Dc - if (Dc==2) - p4est_lnodes_destroy(ptr_pXest_lnodes) - else - p8est_lnodes_destroy(ptr_pXest_lnodes) - end -end - -function pXest_ghost_destroy(::Type{Val{Dc}}, ptr_pXest_ghost) where Dc - if (Dc==2) - p4est_ghost_destroy(ptr_pXest_ghost) - else - p8est_ghost_destroy(ptr_pXest_ghost) - end -end - function pXest_destroy(::Type{Val{Dc}}, ptr_pXest) where Dc if (Dc==2) p4est_destroy(ptr_pXest) @@ -69,6 +53,52 @@ function p4est_get_quadrant_vertex_coordinates(connectivity::Ptr{p4est_connectiv end + +function p6est_get_quadrant_vertex_coordinates(connectivity::Ptr{p6est_connectivity_t}, + treeid::p4est_topidx_t, + x::p4est_qcoord_t, + y::p4est_qcoord_t, + z::p4est_qcoord_t, + xylevel::Int8, + zlevel::Int8, + corner::Cint, + pvxyz::Ptr{Cdouble}) + + function p6est_corner_to_p4est_corner(p6est_corner::Cint) + div(p6est_corner,Cint(2)) + end + + function zquadrant_to_zcorner(z, zlevel_quadrant, p6est_corner) + if p6est_corner in (0,2,4,6) + z + elseif p6est_corner in (1,3,5,7) + z + P4EST_QUADRANT_LEN(zlevel) + end + end + + p6est_qcoord_to_vertex(connectivity, + treeid, + x, + y, + zquadrant_to_zcorner(z,zlevel,corner), + pvxyz) + + vxyz=unsafe_wrap(Array, pvxyz, 3) + zcoord=vxyz[3] + + # Always sets the z-coordinate to 0.0! + p4est_get_quadrant_vertex_coordinates(connectivity[].conn4, + treeid, + x, + y, + xylevel, + p6est_corner_to_p4est_corner(corner), + pvxyz) + vxyz[3]=zcoord +end + + + function p8est_get_quadrant_vertex_coordinates(connectivity::Ptr{p8est_connectivity_t}, treeid::p4est_topidx_t, x::p4est_qcoord_t, @@ -343,134 +373,12 @@ function generate_cell_vertex_lids_nlvertices(cell_vertex_gids) end |> tuple_of_arrays end -function generate_node_coordinates(::Type{Val{Dc}}, - cell_vertex_lids, - nlvertices, - ptr_pXest_connectivity, - ptr_pXest, - ptr_pXest_ghost) where Dc - - - PXEST_CORNERS=2^Dc - pXest_ghost = ptr_pXest_ghost[] - pXest = ptr_pXest[] - - # Obtain ghost quadrants - if (Dc==2) - ptr_ghost_quadrants = Ptr{p4est_quadrant_t}(pXest_ghost.ghosts.array) - else - ptr_ghost_quadrants = Ptr{p8est_quadrant_t}(pXest_ghost.ghosts.array) - end - - tree_offsets = unsafe_wrap(Array, pXest_ghost.tree_offsets, pXest_ghost.num_trees+1) - dnode_coordinates=map(cell_vertex_lids,nlvertices) do cell_vertex_lids, nl - node_coordinates=Vector{Point{Dc,Float64}}(undef,nl) - current=1 - vxy=Vector{Cdouble}(undef,Dc) - pvxy=pointer(vxy,1) - cell_lids=cell_vertex_lids.data - for itree=1:pXest_ghost.num_trees - if (Dc==2) - tree = p4est_tree_array_index(pXest.trees, itree-1)[] - else - tree = p8est_tree_array_index(pXest.trees, itree-1)[] - end - for cell=1:tree.quadrants.elem_count - if (Dc==2) - quadrant=p4est_quadrant_array_index(tree.quadrants, cell-1)[] - else - quadrant=p8est_quadrant_array_index(tree.quadrants, cell-1)[] - end - for vertex=1:PXEST_CORNERS - if (Dc==2) - p4est_get_quadrant_vertex_coordinates(ptr_pXest_connectivity, - p4est_topidx_t(itree-1), - quadrant.x, - quadrant.y, - quadrant.level, - Cint(vertex-1), - pvxy) - else - p8est_get_quadrant_vertex_coordinates(ptr_pXest_connectivity, - p4est_topidx_t(itree-1), - quadrant.x, - quadrant.y, - quadrant.z, - quadrant.level, - Cint(vertex-1), - pvxy) - end - - node_coordinates[cell_lids[current]]=Point{Dc,Float64}(vxy...) - current=current+1 - end - end - end - - # Go over ghost cells - for i=1:pXest_ghost.num_trees - for j=tree_offsets[i]:tree_offsets[i+1]-1 - quadrant = ptr_ghost_quadrants[j+1] - for vertex=1:PXEST_CORNERS - if (Dc==2) - p4est_get_quadrant_vertex_coordinates(ptr_pXest_connectivity, - p4est_topidx_t(i-1), - quadrant.x, - quadrant.y, - quadrant.level, - Cint(vertex-1), - pvxy) - else - p8est_get_quadrant_vertex_coordinates(ptr_pXest_connectivity, - p4est_topidx_t(i-1), - quadrant.x, - quadrant.y, - quadrant.z, - quadrant.level, - Cint(vertex-1), - pvxy) - - end - node_coordinates[cell_lids[current]]=Point{Dc,Float64}(vxy...) - current=current+1 - end - end - end - node_coordinates - end -end - -function generate_grid_and_topology(::Type{Val{Dc}}, - cell_vertex_lids, - nlvertices, - node_coordinates) where {Dc} - grid,topology= - map(cell_vertex_lids,nlvertices,node_coordinates) do cell_vertex_lids, nl, node_coordinates - polytope= Dc==2 ? QUAD : HEX - scalar_reffe=Gridap.ReferenceFEs.ReferenceFE(polytope,Gridap.ReferenceFEs.lagrangian,Float64,1) - cell_types=collect(Fill(1,length(cell_vertex_lids))) - cell_reffes=[scalar_reffe] - cell_vertex_lids_gridap=Gridap.Arrays.Table(cell_vertex_lids.data,cell_vertex_lids.ptrs) - grid = Gridap.Geometry.UnstructuredGrid(node_coordinates, - cell_vertex_lids_gridap, - cell_reffes, - cell_types, - Gridap.Geometry.NonOriented()) - - topology = Gridap.Geometry.UnstructuredGridTopology(node_coordinates, - cell_vertex_lids_gridap, - cell_types, - map(Gridap.ReferenceFEs.get_polytope, cell_reffes), - Gridap.Geometry.NonOriented()) - grid,topology - end |> tuple_of_arrays - grid,topology -end const ITERATOR_RESTRICT_TO_BOUNDARY=Cint(100) const ITERATOR_RESTRICT_TO_INTERIOR=Cint(101) -function generate_face_labeling(parts, +function generate_face_labeling(pXest_type::P4P8estType, + parts, cell_prange, coarse_discrete_model::DiscreteModel{Dc,Dp}, topology, @@ -918,25 +826,29 @@ function setup_distributed_discrete_model(::Type{Val{Dc}}, ptr_pXest, ptr_pXest_ghost, ptr_pXest_lnodes) where Dc + cell_prange = setup_cell_prange(Val{Dc},parts,ptr_pXest,ptr_pXest_ghost) cell_vertex_gids=generate_cell_vertex_gids(ptr_pXest_lnodes,cell_prange) cell_vertex_lids,nlvertices=generate_cell_vertex_lids_nlvertices(cell_vertex_gids) - node_coordinates=generate_node_coordinates(Val{Dc}, + pXest_type = _dim_to_pXest_type(Dc) + + node_coordinates=generate_node_coordinates(pXest_type, cell_vertex_lids, nlvertices, ptr_pXest_connectivity, ptr_pXest, ptr_pXest_ghost) - grid,topology=generate_grid_and_topology(Val{Dc}, + grid,topology=generate_grid_and_topology(pXest_type, cell_vertex_lids, nlvertices, node_coordinates) - face_labeling=generate_face_labeling(parts, + face_labeling=generate_face_labeling(pXest_type, + parts, cell_prange, coarse_discrete_model, topology, diff --git a/test.jl b/test.jl new file mode 100644 index 0000000..5c11c4f --- /dev/null +++ b/test.jl @@ -0,0 +1,33 @@ +using Gridap +using GridapP4est +using MPI +using PartitionedArrays +using GridapDistributed +using Logging + +debug_logger = ConsoleLogger(stderr, Logging.Debug) +global_logger(debug_logger); # Enable the debug logger globally + +MPI.Init() + +ranks = with_mpi() do distribute + distribute(LinearIndices((prod(MPI.Comm_size(MPI.COMM_WORLD)),))) +end + +coarse_model=CartesianDiscreteModel((0,1,0,1),(1,1)) + +model=AnisotropicallyAdapted3DDistributedDiscreteModel(ranks,coarse_model,1,1) + +writevtk(model, "model"); + +ref_coarse_flags=map(ranks,partition(get_cell_gids(model.dmodel))) do rank,indices + flags=zeros(Int,length(indices)) + flags.=nothing_flag + flags[1]=refine_flag + flags[end]=refine_flag + flags +end + +model,glue=GridapP4est.vertically_adapt(model,ref_coarse_flags); + +writevtk(Triangulation(model), "trian"); \ No newline at end of file diff --git a/test/PoissonNonConformingOctreeModelsTests.jl b/test/PoissonNonConformingOctreeModelsTests.jl index 4cb62bd..3c9fdfa 100644 --- a/test/PoissonNonConformingOctreeModelsTests.jl +++ b/test/PoissonNonConformingOctreeModelsTests.jl @@ -271,10 +271,10 @@ module PoissonNonConformingOctreeModelsTests end function run(distribute) - # debug_logger = ConsoleLgger(stderr, Logging.Debug) - # global_logger(debug_logger); # Enable the debug logger globally + #debug_logger = ConsoleLogger(stderr, Logging.Debug) + #global_logger(debug_logger); # Enable the debug logger globally ranks = distribute(LinearIndices((MPI.Comm_size(MPI.COMM_WORLD),))) - for Dc=2:3, perm=1:4, order=1:4, scalar_or_vector in (:scalar,) + for Dc=3:3, perm=1:4, order=1:4, scalar_or_vector in (:scalar,) test(ranks,Val{Dc},perm,order,_field_type(Val{Dc}(),scalar_or_vector)) end for Dc=2:3, perm in (1,2), order in (1,4), scalar_or_vector in (:vector,) From 06ba2740eb87a2ef6fa1bd568342aab0692ca1d1 Mon Sep 17 00:00:00 2001 From: "Alberto F. Martin" Date: Sat, 24 Feb 2024 12:23:12 +1100 Subject: [PATCH 02/29] At this point vertical refinement seems to be working in serial --- Project.toml | 1 - ...callyAdapted3DDistributedDiscreteModels.jl | 15 +- src/FESpaces.jl | 308 ++++++++++++++---- src/OctreeDistributedDiscreteModels.jl | 255 ++++----------- src/PXestTypeMethods.jl | 230 ++++++++++++- ...mlyRefinedForestOfOctreesDiscreteModels.jl | 140 +------- test.jl | 33 -- test/PoissonAnisotropicOctreeModelsTests.jl | 271 +++++++++++++++ .../PoissonAnisotropicOctreeModelsTests.jl | 17 + 9 files changed, 829 insertions(+), 441 deletions(-) delete mode 100644 test.jl create mode 100644 test/PoissonAnisotropicOctreeModelsTests.jl create mode 100644 test/mpi/PoissonAnisotropicOctreeModelsTests.jl diff --git a/Project.toml b/Project.toml index 08108fe..12cf8ec 100644 --- a/Project.toml +++ b/Project.toml @@ -18,7 +18,6 @@ Test = "8dfed614-e22c-5e08-85e1-65c5234f0b40" ArgParse = "1" FillArrays = "0.8.4, 0.9, 0.10, 0.11, 0.12, 1" Gridap = "0.17.22" -GridapDistributed = "0.3.1" MPI = "0.20" P4est_wrapper = "0.2.0" PartitionedArrays = "0.3.3" diff --git a/src/AnisotropicallyAdapted3DDistributedDiscreteModels.jl b/src/AnisotropicallyAdapted3DDistributedDiscreteModels.jl index 1352bdf..25a2064 100644 --- a/src/AnisotropicallyAdapted3DDistributedDiscreteModels.jl +++ b/src/AnisotropicallyAdapted3DDistributedDiscreteModels.jl @@ -37,6 +37,7 @@ function AnisotropicallyAdapted3DDistributedDiscreteModel( ptr_pXest_connectivity, ptr_pXest, pXest_type, + nothing, true, nothing) end @@ -46,7 +47,7 @@ function _vertically_refine_coarsen_balance!(model::OctreeDistributedDiscreteMod pXest_type = model.pXest_type init_fn_callback_c = p6est_vertically_adapt_reset_callbacks() - #coarsen_fn_callback_c = p6est_vertically_coarsen_callbacks() + coarsen_fn_callback_c = p6est_vertically_coarsen_callbacks() refine_callback_c,refine_replace_callback_c = p6est_vertically_refine_callbacks() map(model.dmodel.models,refinement_and_coarsening_flags) do lmodel, flags @@ -62,7 +63,7 @@ function _vertically_refine_coarsen_balance!(model::OctreeDistributedDiscreteMod p6est_vertically_refine!(ptr_new_pXest, refine_callback_c, refine_replace_callback_c) - #pXest_vertically_coarsen!(ptr_new_pXest, coarsen_fn_callback_c) + p6est_vertically_coarsen!(ptr_new_pXest, coarsen_fn_callback_c) pXest_balance!(pXest_type, ptr_new_pXest) p6est_vertically_adapt_update_flags!(model.ptr_pXest,ptr_new_pXest) ptr_new_pXest @@ -95,8 +96,9 @@ function vertically_adapt(model::OctreeDistributedDiscreteModel{Dc,Dp}, pXest_ghost_destroy(model.pXest_type,ptr_pXest_ghost) pXest_lnodes_destroy(model.pXest_type,ptr_pXest_lnodes) + pXest_refinement_rule_type = PXestVerticalRefinementRuleType() adaptivity_glue = _compute_fine_to_coarse_model_glue(model.pXest_type, - PXestVerticalRefinementRuleType(), + pXest_refinement_rule_type, model.parts, model.dmodel, fmodel, @@ -115,6 +117,7 @@ function vertically_adapt(model::OctreeDistributedDiscreteModel{Dc,Dp}, model.ptr_pXest_connectivity, ptr_new_pXest, model.pXest_type, + pXest_refinement_rule_type, false, model) return ref_model, adaptivity_glue @@ -448,6 +451,12 @@ face_labeling = face_labeling=Gridap.Geometry.FaceLabeling(d_to_dface_to_entity, tag_to_entities, tag_to_name) + + add_tag_from_tags!(face_labeling, + "boundary", + ["bottom_boundary", "intermediate_boundary", "top_boundary"]) + + face_labeling end face_labeling end diff --git a/src/FESpaces.jl b/src/FESpaces.jl index 35c7747..ba64ac4 100644 --- a/src/FESpaces.jl +++ b/src/FESpaces.jl @@ -1,13 +1,4 @@ - -function _build_constraint_coefficients_matrix_in_ref_space(Dc, reffe::Tuple{<:Lagrangian,Any,Any}) - cell_polytope = Dc == 2 ? QUAD : HEX - basis, reffe_args, reffe_kwargs = reffe - cell_reffe = ReferenceFE(cell_polytope, basis, reffe_args...; reffe_kwargs...) - - # TO-DO: How can modelH be created such that it is tailored to cell_polytope? - modelH= _generate_unit_hypercube_model(Dc) - modelh=refine(modelH,2) - +function _generate_ref_constraints(modelH,modelh,cell_reffe) VH=TestFESpace(modelH,cell_reffe) Vh=TestFESpace(modelh,cell_reffe) @@ -21,13 +12,76 @@ function _build_constraint_coefficients_matrix_in_ref_space(Dc, reffe::Tuple{<:L cache_cell_dof_ids = array_cache(cell_dof_ids) cache_ref_constraints_contribs = array_cache(ref_constraints_contribs) for cell=1:length(cell_dof_ids) - current_cell_dof_ids=getindex!(cache_cell_dof_ids,cell_dof_ids,cell) - current_ref_constraints_contribs=getindex!(cache_ref_constraints_contribs,ref_constraints_contribs,cell) - ref_constraints[current_cell_dof_ids,:]=current_ref_constraints_contribs + current_cell_dof_ids=getindex!(cache_cell_dof_ids,cell_dof_ids,cell) + current_ref_constraints_contribs=getindex!(cache_ref_constraints_contribs,ref_constraints_contribs,cell) + ref_constraints[current_cell_dof_ids,:]=current_ref_constraints_contribs end ref_constraints end +function _generate_unit_hypercube_model(Dc) + @assert Dc==2 || Dc==3 + if (Dc==2) + modelH=CartesianDiscreteModel((0,1,0,1),(1,1)) + else + modelH=CartesianDiscreteModel((0,1,0,1,0,1),(1,1,1)) + end + modelH +end + +function _build_constraint_coefficients_matrix_in_ref_space(::PXestUniformRefinementRuleType, + Dc, + reffe::Tuple{<:Lagrangian,Any,Any}) + cell_polytope = Dc == 2 ? QUAD : HEX + basis, reffe_args, reffe_kwargs = reffe + cell_reffe = ReferenceFE(cell_polytope, basis, reffe_args...; reffe_kwargs...) + modelH=_generate_unit_hypercube_model(Dc) + modelh=refine(modelH,2) + _generate_ref_constraints(modelH,modelh,cell_reffe) +end + +function _build_constraint_coefficients_matrix_in_ref_space(::PXestVerticalRefinementRuleType, + Dc, + reffe::Tuple{<:Lagrangian,Any,Any}) + @assert Dc==3 + cell_polytope = HEX + basis, reffe_args, reffe_kwargs = reffe + cell_reffe = ReferenceFE(cell_polytope, basis, reffe_args...; reffe_kwargs...) + modelH= _generate_unit_hypercube_model(Dc) + modelh=refine(modelH,(2,1,1)) + return _generate_ref_constraints(modelH,modelh,cell_reffe) + + # ref_constraints=Vector{Vector{Matrix{Float64}}}(undef,Dc-1) + # # Edges + # ref_constraints[1]=Vector{Matrix{Float64}}(undef,12) + # for (i,partition) in enumerate(((2,1,1),(2,1,1),(2,1,1))) + # modelh=refine(modelH,partition) + # for j in (2*(i-1)+1,2*i) + # ref_constraints[1][j]=_generate_ref_constraints(modelH,modelh,cell_reffe) + # end + # end + # # Faces + # ref_constraints[2]=Vector{Matrix{Float64}}(undef,6) + # for (i,partition) in enumerate(((1,1,2),(1,2,1),(2,1,1))) + # modelh=refine(modelH,partition) + # for j in (2*i-1,2*i) + # ref_constraints[2][j]=_generate_ref_constraints(modelH,modelh,cell_reffe) + # end + # end + # ref_constraints +end + +function _allocate_face_subface_ldof_to_cell_ldof(num_faces, num_subfaces, num_dofs_x_face) + face_subface_ldof_to_cell_ldof = Vector{Vector{Vector{Int32}}}(undef,num_faces) + for face=1:num_faces + face_subface_ldof_to_cell_ldof[face]=Vector{Vector{Int32}}(undef,num_subfaces) + for subface=1:num_subfaces + face_subface_ldof_to_cell_ldof[face][subface]=Vector{Int32}(undef,num_dofs_x_face) + end + end + face_subface_ldof_to_cell_ldof +end + function _fill_face_subface_ldof_to_cell_ldof!(face_subface_ldof_to_cell_ldof, num_faces, coarse_faces_to_child_ids, @@ -46,16 +100,15 @@ function _fill_face_subface_ldof_to_cell_ldof!(face_subface_ldof_to_cell_ldof, end end - -function _generate_face_subface_ldof_to_cell_ldof(Df,Dc,reffe::Tuple{<:Lagrangian,Any,Any}) +function _generate_face_subface_ldof_to_cell_ldof(ref_rule::PXestUniformRefinementRuleType, + Df,Dc,reffe::Tuple{<:Lagrangian,Any,Any}) + cell_polytope = (Dc == 2) ? QUAD : HEX - coarse_faces_to_child_ids = (Dc == 2) ? _coarse_faces_to_child_ids_2D : - _coarse_faces_to_child_ids_3D + _coarse_faces_to_child_ids = coarse_faces_to_child_ids(ref_rule,Df,Dc) basis, reffe_args, reffe_kwargs = reffe cell_reffe = ReferenceFE(cell_polytope, basis, reffe_args...; reffe_kwargs...) - # TO-DO: How can modelH be created such that it is tailored to cell_polytope? modelH= _generate_unit_hypercube_model(Dc) modelh=refine(modelH,2) Vh=TestFESpace(modelh,reffe) @@ -71,7 +124,7 @@ function _generate_face_subface_ldof_to_cell_ldof(Df,Dc,reffe::Tuple{<:Lagrangia face_subface_ldof_to_cell_ldof=_allocate_face_subface_ldof_to_cell_ldof(num_faces,num_subfaces,num_dofs_x_face) _fill_face_subface_ldof_to_cell_ldof!(face_subface_ldof_to_cell_ldof, num_faces, - coarse_faces_to_child_ids, + _coarse_faces_to_child_ids, face_dofs, cells_dof_ids, first_face) @@ -83,54 +136,179 @@ function _generate_face_subface_ldof_to_cell_ldof(Df,Dc,reffe::Tuple{<:Lagrangia edge_subedge_ldof_to_cell_ldof=_allocate_face_subface_ldof_to_cell_ldof(num_edges,num_subedges,num_dofs_x_face) _fill_face_subface_ldof_to_cell_ldof!(edge_subedge_ldof_to_cell_ldof, num_edges, - _coarse_edges_to_child_ids_3D, - face_dofs, - cells_dof_ids, - first_face) + _coarse_faces_to_child_ids, + face_dofs, + cells_dof_ids, + first_face) edge_subedge_ldof_to_cell_ldof end end +function _generate_face_subface_ldof_to_cell_ldof(ref_rule::PXestVerticalRefinementRuleType, + Df,Dc,reffe::Tuple{<:Lagrangian,Any,Any}) + + cell_polytope = HEX + _coarse_faces_to_child_ids = coarse_faces_to_child_ids(ref_rule,Df,Dc) + + basis, reffe_args, reffe_kwargs = reffe + cell_reffe = ReferenceFE(cell_polytope, basis, reffe_args...; reffe_kwargs...) + + modelH=_generate_unit_hypercube_model(Dc) + modelh=refine(modelH,(2,1,1)) + Vh=TestFESpace(modelh,reffe) + cells_dof_ids=get_cell_dof_ids(Vh) + + first_face = get_offset(get_polytope(cell_reffe),Df) + face_dofs = get_face_dofs(cell_reffe) + num_dofs_x_face = length(face_dofs[first_face+1]) + + if (Df==1 && Dc==3) + num_edges=12 + num_subedges=2 + num_faces=2*Dc + edge_subedge_ldof_to_cell_ldof= + _allocate_face_subface_ldof_to_cell_ldof(num_edges,num_subedges,num_dofs_x_face) + _fill_face_subface_ldof_to_cell_ldof!(edge_subedge_ldof_to_cell_ldof, + num_edges, + _coarse_faces_to_child_ids, + face_dofs, + cells_dof_ids, + first_face) + return edge_subedge_ldof_to_cell_ldof + else + @assert Df==2 && Dc==3 + num_faces = 2*Dc + num_subfaces = 2 + face_subface_ldof_to_cell_ldof = + _allocate_face_subface_ldof_to_cell_ldof(num_faces,num_subfaces,num_dofs_x_face) + + _fill_face_subface_ldof_to_cell_ldof!(face_subface_ldof_to_cell_ldof, + num_faces, + _coarse_faces_to_child_ids, + face_dofs, + cells_dof_ids, + first_face) + + return face_subface_ldof_to_cell_ldof + + # for (i,partition) in enumerate(((1,1,2),(1,2,1),(2,1,1))) + # modelh=refine(modelH,partition) + # Vh=TestFESpace(modelh,reffe) + # cells_dof_ids=get_cell_dof_ids(Vh) + + # for j in (2*i-1,2*i) + # face_subface_ldof_to_cell_ldof[j]= + # _allocate_face_subface_ldof_to_cell_ldof(num_faces,num_subfaces,num_dofs_x_face) + + # _fill_face_subface_ldof_to_cell_ldof!(face_subface_ldof_to_cell_ldof[j], + # num_faces, + # _coarse_faces_to_child_ids, + # face_dofs, + # cells_dof_ids, + # first_face) + # end + # face_subface_ldof_to_cell_ldof + # end + end + + + # if (Df==1 && Dc==3) + # num_edges=12 + # num_subedges=2 + # num_faces=2*Dc + # edge_subedge_ldof_to_cell_ldof = Vector{Vector{Vector{Vector{Int32}}}}(undef,num_edges) + # for (i,partition) in enumerate(((2,1,1),(1,2,1),(1,1,2))) + # modelh=refine(modelH,partition) + # Vh=TestFESpace(modelh,reffe) + # cells_dof_ids=get_cell_dof_ids(Vh) + # for j in (2*(i-1)+1,2*i) + # println("YYY i=$(i) j=$(j)") + # edge_subedge_ldof_to_cell_ldof[j]= + # _allocate_face_subface_ldof_to_cell_ldof(num_edges,num_subedges,num_dofs_x_face) + # _fill_face_subface_ldof_to_cell_ldof!(edge_subedge_ldof_to_cell_ldof[j], + # num_edges, + # _coarse_faces_to_child_ids, + # face_dofs, + # cells_dof_ids, + # first_face) + # end + # end + # return edge_subedge_ldof_to_cell_ldof + # else + # @assert Df==2 && Dc==3 + # num_faces = 2*Dc + # num_subfaces = 2 + # face_subface_ldof_to_cell_ldof = Vector{Vector{Vector{Vector{Int32}}}}(undef,num_faces) + + # for (i,partition) in enumerate(((1,1,2),(1,2,1),(2,1,1))) + # modelh=refine(modelH,partition) + # Vh=TestFESpace(modelh,reffe) + # cells_dof_ids=get_cell_dof_ids(Vh) + + # for j in (2*i-1,2*i) + # face_subface_ldof_to_cell_ldof[j]= + # _allocate_face_subface_ldof_to_cell_ldof(num_faces,num_subfaces,num_dofs_x_face) + + # _fill_face_subface_ldof_to_cell_ldof!(face_subface_ldof_to_cell_ldof[j], + # num_faces, + # _coarse_faces_to_child_ids, + # face_dofs, + # cells_dof_ids, + # first_face) + # end + # face_subface_ldof_to_cell_ldof + # end + # return face_subface_ldof_to_cell_ldof + # end +end + const _coarse_faces_to_child_ids_2D=[1 2; 3 4; 1 3; 2 4] const _coarse_faces_to_child_ids_3D=[1 2 3 4; 5 6 7 8; 1 2 5 6; 3 4 7 8; 1 3 5 7; 2 4 6 8; ] const _coarse_edges_to_child_ids_3D=[1 2; 3 4; 5 6; 7 8; 1 3; 2 4; 5 7; 6 8; 1 5; 2 6; 3 7; 4 8; ] +const _coarse_faces_to_child_ids_vertical_refinement=[1 2; 1 2; 1 2; 1 2; 1 1; 2 2] +const _coarse_edges_to_child_ids_vertical_refinement=[1 2; 1 2; 1 2; 1 2; + 1 1; 2 2; 1 1; 2 2; + 1 1; 2 2; 1 1; 2 2;] -function _generate_unit_hypercube_model(Dc) - @assert Dc==2 || Dc==3 +function coarse_faces_to_child_ids(::PXestUniformRefinementRuleType,Df,Dc) + if (Df==Dc-1) if (Dc==2) - modelH=CartesianDiscreteModel((0,1,0,1),(1,1)) + _coarse_faces_to_child_ids_2D else - modelH=CartesianDiscreteModel((0,1,0,1,0,1),(1,1,1)) + _coarse_faces_to_child_ids_3D end - modelH -end + else + @assert Df==1 && Dc==3 + _coarse_edges_to_child_ids_3D + end +end -function _allocate_face_subface_ldof_to_cell_ldof(num_faces, num_subfaces, num_dofs_x_face) - face_subface_ldof_to_cell_ldof = Vector{Vector{Vector{Int}}}(undef,num_faces) - for face=1:num_faces - face_subface_ldof_to_cell_ldof[face]=Vector{Vector{Int}}(undef,num_subfaces) - for subface=1:num_subfaces - face_subface_ldof_to_cell_ldof[face][subface]=Vector{Int}(undef,num_dofs_x_face) - end - end - face_subface_ldof_to_cell_ldof -end +function coarse_faces_to_child_ids(::PXestVerticalRefinementRuleType,Df,Dc) + @assert Dc==3 + if (Df==2) + _coarse_faces_to_child_ids_vertical_refinement + else + @assert Df==1 && Dc==3 + _coarse_edges_to_child_ids_vertical_refinement + end +end -function _generate_face_subface_ldof_to_cell_ldof(Df,Dc,reffe::Tuple{<:RaviartThomas,Any,Any}) +function _generate_face_subface_ldof_to_cell_ldof(ref_rule::PXestUniformRefinementRuleType, + Df,Dc,reffe::Tuple{<:RaviartThomas,Any,Any}) cell_polytope = (Dc == 2) ? QUAD : HEX - coarse_faces_to_child_ids = (Dc == 2) ? _coarse_faces_to_child_ids_2D : - _coarse_faces_to_child_ids_3D + + _coarse_faces_to_child_ids = coarse_faces_to_child_ids(ref_rule,Df,Dc) if (Df==Dc-1) # Facets basis, reffe_args, reffe_kwargs = reffe cell_reffe = ReferenceFE(cell_polytope, basis, reffe_args...; reffe_kwargs...) - # TO-DO: How can modelH be created such that it is tailored to cell_polytope? modelH= _generate_unit_hypercube_model(Dc) modelh=refine(modelH,2) + RTh=TestFESpace(modelh,reffe) num_faces = 2*Dc num_subfaces = 2^(Dc-1) @@ -141,7 +319,7 @@ function _generate_face_subface_ldof_to_cell_ldof(Df,Dc,reffe::Tuple{<:RaviartTh RTh_cell_dof_ids=get_cell_dof_ids(RTh) _fill_face_subface_ldof_to_cell_ldof!(face_subface_ldof_to_cell_ldof, num_faces, - coarse_faces_to_child_ids, + _coarse_faces_to_child_ids, face_own_dofs, RTh_cell_dof_ids, first_face) @@ -157,32 +335,17 @@ function _generate_face_subface_ldof_to_cell_ldof(Df,Dc,reffe::Tuple{<:RaviartTh face_subface_ldof_to_cell_ldof end -function _build_constraint_coefficients_matrix_in_ref_space(Dc, reffe::Tuple{<:RaviartThomas,Any,Any}) +function _build_constraint_coefficients_matrix_in_ref_space(::PXestUniformRefinementRuleType, + Dc, + reffe::Tuple{<:RaviartThomas,Any,Any}) cell_polytope = Dc == 2 ? QUAD : HEX basis, reffe_args, reffe_kwargs = reffe cell_reffe = ReferenceFE(cell_polytope, basis, reffe_args...; reffe_kwargs...) - # TO-DO: How can modelH be created such that it is tailored to cell_polytope? modelH= _generate_unit_hypercube_model(Dc) modelh=refine(modelH,2) - - VH=TestFESpace(modelH,cell_reffe) - Vh=TestFESpace(modelh,cell_reffe) - - uH=get_fe_basis(VH) - uHh=change_domain(uH,get_triangulation(modelh),ReferenceDomain()) - σRTh=Gridap.FESpaces.get_fe_dof_basis(Vh) - ref_constraints_contribs=σRTh(uHh) - ref_constraints = Matrix{Float64}(undef,num_free_dofs(Vh),num_free_dofs(VH)) - cell_dof_ids = get_cell_dof_ids(Vh) - cache_cell_dof_ids = array_cache(cell_dof_ids) - cache_ref_constraints_contribs = array_cache(ref_constraints_contribs) - for cell=1:length(cell_dof_ids) - current_cell_dof_ids=getindex!(cache_cell_dof_ids,cell_dof_ids,cell) - current_ref_constraints_contribs=getindex!(cache_ref_constraints_contribs,ref_constraints_contribs,cell) - ref_constraints[current_cell_dof_ids,:]=current_ref_constraints_contribs - end + ref_constraints=_generate_ref_constraints(modelH,modelh,cell_reffe) # We change the sign of the constraints coefficients here so that # the unit normal to hanging faces matches the unit normal # of the owner face. This way, we properly glue the global DoFs at @@ -375,6 +538,7 @@ function _generate_constraints!(Df, hanging_lvertex_within_first_subface = 2^oface_dim cur_subface_own_dofs = subface_own_dofs[oface_dim][hanging_lvertex_within_first_subface] elseif (Df == 1 && Dc == 3) # Am I an edge? + @debug "Df=$(Df) Dc=$(Dc) cell=$(cell) lface=$(lface) ocell=$(ocell) ocell_lface=$(ocell_lface) oface_dim=$(oface_dim) subface=$(subface) subface_own_dofs[oface_dim]=$(subface_own_dofs[oface_dim])" if (subface < 0) # Edge hanging in the interior of a face @assert subface == -1 || subface == -2 || subface == -3 || subface == -4 @assert oface_dim == Dc - 1 @@ -408,7 +572,7 @@ function _generate_constraints!(Df, oface = getindex!(cache_cell_faces[oface_dim+1], cell_faces[oface_dim+1], ocell)[ocell_lface_within_dim] oface_lid, _ = owner_faces_lids[oface_dim][oface] pindex = owner_faces_pindex[oface_dim][oface_lid] - @debug "Df=$(Df) Dc=$(Dc) fid_hanging=$(fid_hanging) cell=$(cell) oface=$(oface) oface_lid=$(oface_lid) pindex=$(pindex) ocell=$(ocell) ocell_lface=$(ocell_lface) subface=$(subface) oface_dim=$(oface_dim) cur_subface_own_dofs=$(cur_subface_own_dofs) face_own_dofs=$(face_own_dofs[offset+lface])" + @debug "Df=$(Df) Dc=$(Dc) fid_hanging=$(fid_hanging) cell=$(cell) oface=$(oface) oface_lid=$(oface_lid) pindex=$(pindex) ocell=$(ocell) ocell_lface=$(ocell_lface) ocell_lface_within_dim=$(ocell_lface_within_dim) subface=$(subface) oface_dim=$(oface_dim) cur_subface_own_dofs=$(cur_subface_own_dofs) face_own_dofs=$(face_own_dofs[offset+lface])" for ((ldof, dof), ldof_subface) in zip(enumerate(face_own_dofs[offset+lface]), cur_subface_own_dofs) push!(sDOF_to_dof, current_cell_dof_ids[dof]) push!(sDOF_to_dofs, hanging_faces_owner_face_dofs[fid_hanging]) @@ -417,8 +581,9 @@ function _generate_constraints!(Df, for (ifdof, icdof) in enumerate(face_dofs[ocell_lface]) pifdof = node_permutations[oface_dim][pindex][ifdof] ldof_coarse = face_dofs[ocell_lface][pifdof] - coeffs[ifdof] = - ref_constraints[face_subface_ldof_to_cell_ldof[oface_dim][ocell_lface_within_dim][subface][ldof_subface], ldof_coarse] + @debug "ldof_coarse=$(ldof_coarse) ifdof=$(ifdof) icdof=$(icdof) pifdof=$(pifdof) subface=$(subface) ldof_subface=$(ldof_subface)" + ldof_fine=face_subface_ldof_to_cell_ldof[oface_dim][ocell_lface_within_dim][subface][ldof_subface] + coeffs[ifdof] = ref_constraints[ldof_fine, ldof_coarse] end push!(sDOF_to_coeffs, coeffs) end @@ -896,7 +1061,7 @@ function _is_conforming(model::OctreeDistributedDiscreteModel) reduction(&,is_local_conforming,init=true,destination=:all).item_ref[] end -function _add_constraints(model::GridapDistributed.DistributedDiscreteModel{Dc}, +function _add_constraints(model::OctreeDistributedDiscreteModel{Dc}, reffe, spaces_wo_constraints; conformity=nothing, @@ -907,12 +1072,15 @@ function _add_constraints(model::GridapDistributed.DistributedDiscreteModel{Dc}, local_cell_dof_ids=map(get_cell_dof_ids,spaces_w_constraints) else @assert conformity==nothing || conformity!=:L2 - ref_constraints = _build_constraint_coefficients_matrix_in_ref_space(Dc, reffe) + ref_constraints = _build_constraint_coefficients_matrix_in_ref_space(model.pXest_refinement_rule_type, + Dc, + reffe) face_subface_ldof_to_cell_ldof = Vector{Vector{Vector{Vector{Int32}}}}(undef, Dc-1) - face_subface_ldof_to_cell_ldof[Dc-1] = _generate_face_subface_ldof_to_cell_ldof(Dc-1, Dc, reffe) + face_subface_ldof_to_cell_ldof[Dc-1] = + _generate_face_subface_ldof_to_cell_ldof(model.pXest_refinement_rule_type, Dc-1, Dc, reffe) if (Dc == 3) face_subface_ldof_to_cell_ldof[1] = - _generate_face_subface_ldof_to_cell_ldof(1, Dc, reffe) + _generate_face_subface_ldof_to_cell_ldof(model.pXest_refinement_rule_type, 1, Dc, reffe) end sDOF_to_dof, sDOF_to_dofs, sDOF_to_coeffs = generate_constraints(model, spaces_wo_constraints, reffe, ref_constraints, face_subface_ldof_to_cell_ldof) diff --git a/src/OctreeDistributedDiscreteModels.jl b/src/OctreeDistributedDiscreteModels.jl index 1a0012d..d8b3094 100644 --- a/src/OctreeDistributedDiscreteModels.jl +++ b/src/OctreeDistributedDiscreteModels.jl @@ -41,6 +41,7 @@ mutable struct OctreeDistributedDiscreteModel{Dc,Dp,A,B,C,D,E,F} <: GridapDistri ptr_pXest_connectivity :: E ptr_pXest :: F pXest_type :: PXestType + pXest_refinement_rule_type :: Union{Nothing,PXestRefinementRuleType} # The model for which this variable is true, is the one # ultimately responsible for deallocating the pXest_connectivity @@ -60,6 +61,7 @@ mutable struct OctreeDistributedDiscreteModel{Dc,Dp,A,B,C,D,E,F} <: GridapDistri ptr_pXest_connectivity, ptr_pXest, pXest_type::PXestType, + pXest_refinement_rule_type::Union{Nothing,PXestRefinementRuleType}, owns_ptr_pXest_connectivity::Bool, gc_ref) @@ -81,6 +83,7 @@ mutable struct OctreeDistributedDiscreteModel{Dc,Dp,A,B,C,D,E,F} <: GridapDistri ptr_pXest_connectivity, ptr_pXest, pXest_type, + pXest_refinement_rule_type, owns_ptr_pXest_connectivity, gc_ref) Init(model) @@ -96,6 +99,7 @@ function OctreeDistributedDiscreteModel( ptr_pXest_connectivity, ptr_pXest, pXest_type, + pXest_refinement_rule_type, owns_ptr_pXest_connectivity, gc_ref) where {Dc,Dp} @@ -108,6 +112,7 @@ function OctreeDistributedDiscreteModel( ptr_pXest_connectivity, ptr_pXest, pXest_type, + pXest_refinement_rule_type, owns_ptr_pXest_connectivity, gc_ref) end @@ -128,11 +133,11 @@ function OctreeDistributedDiscreteModel(parts::AbstractVector{<:Integer}, ptr_pXest_connectivity, ptr_pXest, ptr_pXest_ghost, - ptr_pXest_lnodes = setup_ptr_pXest_objects(Val{Dc}, - comm, - coarse_model, - num_uniform_refinements) - dmodel = setup_distributed_discrete_model(Val{Dc}, + ptr_pXest_lnodes = setup_ptr_pXest_objects(pXest_type, + comm, + coarse_model, + num_uniform_refinements) + dmodel = setup_distributed_discrete_model(pXest_type, parts, coarse_model, ptr_pXest_connectivity, @@ -153,6 +158,7 @@ function OctreeDistributedDiscreteModel(parts::AbstractVector{<:Integer}, ptr_pXest_connectivity, ptr_pXest, pXest_type, + nothing, true, nothing) else @@ -184,6 +190,7 @@ function VoidOctreeDistributedDiscreteModel(coarse_model::DiscreteModel{Dc,Dp},p ptr_pXest_connectivity, nothing, _dim_to_pXest_type(Dc), + nothing, true, nothing) end @@ -198,6 +205,7 @@ function VoidOctreeDistributedDiscreteModel(model::OctreeDistributedDiscreteMode model.ptr_pXest_connectivity, nothing, _dim_to_pXest_type(Dc), + nothing, false, model) end @@ -233,58 +241,13 @@ function Init(a::OctreeDistributedDiscreteModel) end function Finalize(a::OctreeDistributedDiscreteModel) - octree_distributed_discrete_model_free!(a) + # octree_distributed_discrete_model_free!(a) return nothing end ################################################################### # Private methods -function pXest_partition_given!(::Type{Val{Dc}}, ptr_pXest, new_num_cells_per_part) where Dc - if (Dc==2) - p4est_partition_given(ptr_pXest, new_num_cells_per_part) - else - p8est_partition_given(ptr_pXest, new_num_cells_per_part) - end -end - -function pXest_uniformly_refine!(::Type{Val{Dc}}, ptr_pXest) where Dc - # Refine callbacks - function refine_fn_2d(::Ptr{p4est_t},which_tree::p4est_topidx_t,quadrant::Ptr{p4est_quadrant_t}) - return Cint(1) - end - function refine_fn_3d(::Ptr{p8est_t},which_tree::p4est_topidx_t,quadrant::Ptr{p8est_quadrant_t}) - return Cint(1) - end - if (Dc==2) - # C-callable refine callback 2D - refine_fn_c = @cfunction($refine_fn_2d,Cint,(Ptr{p4est_t}, p4est_topidx_t, Ptr{p4est_quadrant_t})) - p4est_refine(ptr_pXest, Cint(0), refine_fn_c, C_NULL) - else - # C-callable refine callback 3D - refine_fn_c = @cfunction($refine_fn_3d,Cint,(Ptr{p8est_t}, p4est_topidx_t, Ptr{p8est_quadrant_t})) - p8est_refine(ptr_pXest, Cint(0), refine_fn_c, C_NULL) - end -end - -function pXest_uniformly_coarsen!(::Type{Val{Dc}}, ptr_pXest) where Dc - # Coarsen callbacks - function coarsen_fn_2d(::Ptr{p4est_t},::p4est_topidx_t,::Ptr{Ptr{p4est_quadrant_t}}) - return Cint(1) - end - function coarsen_fn_3d(::Ptr{p8est_t},::p4est_topidx_t,::Ptr{Ptr{p8est_quadrant_t}}) - return Cint(1) - end - if (Dc==2) - # C-callable coasen callback - coarsen_fn_c=@cfunction($coarsen_fn_2d,Cint,(Ptr{p4est_t}, p4est_topidx_t, Ptr{Ptr{p4est_quadrant_t}})) - p4est_coarsen(ptr_pXest, Cint(0), coarsen_fn_c, C_NULL) - else - coarsen_fn_c=@cfunction($coarsen_fn_3d,Cint,(Ptr{p8est_t}, p4est_topidx_t, Ptr{Ptr{p8est_quadrant_t}})) - p8est_coarsen(ptr_pXest, Cint(0), coarsen_fn_c, C_NULL) - end -end - function get_num_children(::P4estType,::PXestUniformRefinementRuleType) 4 end @@ -1225,7 +1188,7 @@ function Gridap.Adaptivity.refine(model::OctreeDistributedDiscreteModel{Dc,Dp}; if (GridapDistributed.i_am_in(old_comm)) # Copy and refine input p4est ptr_new_pXest = pXest_copy(model.pXest_type, model.ptr_pXest) - pXest_uniformly_refine!(Val{Dc}, ptr_new_pXest) + pXest_uniformly_refine!(model.pXest_type, ptr_new_pXest) else ptr_new_pXest = nothing end @@ -1236,7 +1199,7 @@ function Gridap.Adaptivity.refine(model::OctreeDistributedDiscreteModel{Dc,Dp}; if !isa(parts,Nothing) aux = ptr_new_pXest - ptr_new_pXest = _pXest_to_new_comm(Val{Dc}, + ptr_new_pXest = _pXest_to_new_comm(pXest_type, ptr_new_pXest, model.ptr_pXest_connectivity, model.parts.comm, @@ -1247,12 +1210,12 @@ function Gridap.Adaptivity.refine(model::OctreeDistributedDiscreteModel{Dc,Dp}; end # Extract ghost and lnodes - ptr_pXest_ghost = setup_pXest_ghost(Val{Dc}, ptr_new_pXest) - ptr_pXest_lnodes = setup_pXest_lnodes(Val{Dc}, ptr_new_pXest, ptr_pXest_ghost) + ptr_pXest_ghost = setup_pXest_ghost(pXest_type, ptr_new_pXest) + ptr_pXest_lnodes = setup_pXest_lnodes(pXest_type, ptr_new_pXest, ptr_pXest_ghost) # Build fine-grid mesh new_parts = isa(parts,Nothing) ? model.parts : parts - fmodel = setup_distributed_discrete_model(Val{Dc}, + fmodel = setup_distributed_discrete_model(pXest_type, new_parts, model.coarse_model, model.ptr_pXest_connectivity, @@ -1278,6 +1241,7 @@ function Gridap.Adaptivity.refine(model::OctreeDistributedDiscreteModel{Dc,Dp}; model.ptr_pXest_connectivity, ptr_new_pXest, pXest_type, + PXestUniformRefinementRuleType(), false, model) @@ -1331,8 +1295,8 @@ function Gridap.Adaptivity.adapt(model::OctreeDistributedDiscreteModel{Dc,Dp}, ptr_new_pXest = _refine_coarsen_balance!(model, _refinement_and_coarsening_flags) # Extract ghost and lnodes - ptr_pXest_ghost = setup_pXest_ghost(Val{Dc}, ptr_new_pXest) - ptr_pXest_lnodes = setup_pXest_lnodes_nonconforming(Val{Dc}, ptr_new_pXest, ptr_pXest_ghost) + ptr_pXest_ghost = setup_pXest_ghost(model.pXest_type, ptr_new_pXest) + ptr_pXest_lnodes = setup_pXest_lnodes_nonconforming(model.pXest_type, ptr_new_pXest, ptr_pXest_ghost) # Build fine-grid mesh fmodel,non_conforming_glue = setup_non_conforming_distributed_discrete_model(model.pXest_type, @@ -1345,8 +1309,9 @@ function Gridap.Adaptivity.adapt(model::OctreeDistributedDiscreteModel{Dc,Dp}, pXest_ghost_destroy(model.pXest_type,ptr_pXest_ghost) pXest_lnodes_destroy(model.pXest_type,ptr_pXest_lnodes) + pXest_refinement_rule_type = PXestUniformRefinementRuleType() adaptivity_glue = _compute_fine_to_coarse_model_glue(model.pXest_type, - PXestUniformRefinementRuleType(), + pXest_refinement_rule_type, model.parts, model.dmodel, fmodel, @@ -1365,6 +1330,7 @@ function Gridap.Adaptivity.adapt(model::OctreeDistributedDiscreteModel{Dc,Dp}, model.ptr_pXest_connectivity, ptr_new_pXest, model.pXest_type, + pXest_refinement_rule_type, false, model) return ref_model, adaptivity_glue @@ -1375,18 +1341,18 @@ function Gridap.Adaptivity.coarsen(model::OctreeDistributedDiscreteModel{Dc,Dp}) if (GridapDistributed.i_am_in(comm)) # Copy and coarsen input p4est ptr_new_pXest = pXest_copy(model.pXest_type, model.ptr_pXest) - pXest_uniformly_coarsen!(Val{Dc}, ptr_new_pXest) + pXest_uniformly_coarsen!(model.pXest_type, ptr_new_pXest) else ptr_new_pXest=nothing end if (GridapDistributed.i_am_in(comm)) # Extract ghost and lnodes - ptr_pXest_ghost = setup_pXest_ghost(Val{Dc}, ptr_new_pXest) - ptr_pXest_lnodes = setup_pXest_lnodes(Val{Dc}, ptr_new_pXest, ptr_pXest_ghost) + ptr_pXest_ghost = setup_pXest_ghost(model.pXest_type, ptr_new_pXest) + ptr_pXest_lnodes = setup_pXest_lnodes(model.pXest_type, ptr_new_pXest, ptr_pXest_ghost) # Build coarse-grid mesh - cmodel = setup_distributed_discrete_model(Val{Dc}, + cmodel = setup_distributed_discrete_model(model.pXest_type, model.parts, model.coarse_model, model.ptr_pXest_connectivity, @@ -1410,6 +1376,7 @@ function Gridap.Adaptivity.coarsen(model::OctreeDistributedDiscreteModel{Dc,Dp}) model.coarse_model, model.ptr_pXest_connectivity, ptr_new_pXest, + PXestUniformRefinementRuleType(), false, model) return c_octree_model, dglue @@ -1418,63 +1385,22 @@ function Gridap.Adaptivity.coarsen(model::OctreeDistributedDiscreteModel{Dc,Dp}) end end -function pXest_deflate_quadrants(::Type{Val{Dc}},ptr_pXest,data) where Dc - if Dc ==2 - P4est_wrapper.p4est_deflate_quadrants(ptr_pXest,data) - else - P4est_wrapper.p8est_deflate_quadrants(ptr_pXest,data) - end -end - -function pXest_comm_count_pertree(::Type{Val{Dc}},ptr_pXest,pertree) where Dc - if Dc == 2 - p4est_comm_count_pertree(ptr_pXest,pertree) - else - p8est_comm_count_pertree(ptr_pXest,pertree) - end -end -function pXest_inflate(::Type{Val{Dc}}, - comm, - ptr_pXest_conn, - global_first_quadrant, - pertree, - quadrants, - data, - user_pointer) where Dc - if Dc == 2 - P4est_wrapper.p4est_inflate(comm, - ptr_pXest_conn, - global_first_quadrant, - pertree, - quadrants, - data, - user_pointer) - else - P4est_wrapper.p8est_inflate(comm, - ptr_pXest_conn, - global_first_quadrant, - pertree, - quadrants, - data, - user_pointer) - end -end # We have a p4est distributed among P processors. This function # instantiates the same among Q processors. -function _pXest_to_new_comm(::Type{Val{Dc}},ptr_pXest, ptr_pXest_conn, old_comm, new_comm) where Dc +function _pXest_to_new_comm(pXest_type,ptr_pXest, ptr_pXest_conn, old_comm, new_comm) A = is_included(old_comm,new_comm) # old \subset new (smaller to larger nparts) B = is_included(new_comm,old_comm) # old \supset new (larger to smaller nparts) @assert xor(A,B) if (A) - _pXest_to_new_comm_old_subset_new(Val{Dc},ptr_pXest, ptr_pXest_conn, old_comm, new_comm) + _pXest_to_new_comm_old_subset_new(pXest_type,ptr_pXest, ptr_pXest_conn, old_comm, new_comm) else - _pXest_to_new_comm_old_supset_new(Val{Dc},ptr_pXest, ptr_pXest_conn, old_comm, new_comm) + _pXest_to_new_comm_old_supset_new(pXest_type,ptr_pXest, ptr_pXest_conn, old_comm, new_comm) end end -function _pXest_to_new_comm_old_subset_new(::Type{Val{Dc}},ptr_pXest, ptr_pXest_conn, old_comm, new_comm) where Dc +function _pXest_to_new_comm_old_subset_new(pXest_type,ptr_pXest, ptr_pXest_conn, old_comm, new_comm) if (GridapDistributed.i_am_in(new_comm)) new_comm_num_parts = GridapDistributed.num_parts(new_comm) global_first_quadrant = Vector{P4est_wrapper.p4est_gloidx_t}(undef,new_comm_num_parts+1) @@ -1495,15 +1421,15 @@ function _pXest_to_new_comm_old_subset_new(::Type{Val{Dc}},ptr_pXest, ptr_pXest_ global_first_quadrant[i] = old_global_first_quadrant[end] end MPI.Bcast!(global_first_quadrant,0,new_comm) - quadrants = pXest_deflate_quadrants(Val{Dc},ptr_pXest,C_NULL) - pXest_comm_count_pertree(Val{Dc},ptr_pXest,pertree) + quadrants = pXest_deflate_quadrants(pXest_type,ptr_pXest,C_NULL) + pXest_comm_count_pertree(pXest_type,ptr_pXest,pertree) MPI.Bcast!(pertree,0,new_comm) else MPI.Bcast!(global_first_quadrant,0,new_comm) quadrants = sc_array_new_count(sizeof(p4est_quadrant_t), 0) MPI.Bcast!(pertree,0,new_comm) end - return pXest_inflate(Val{Dc}, + return pXest_inflate(pXest_type, new_comm, ptr_pXest_conn, global_first_quadrant, @@ -1516,13 +1442,13 @@ function _pXest_to_new_comm_old_subset_new(::Type{Val{Dc}},ptr_pXest, ptr_pXest_ end end -function _pXest_to_new_comm_old_supset_new(::Type{Val{Dc}},ptr_pXest, ptr_pXest_conn, old_comm, new_comm) where Dc +function _pXest_to_new_comm_old_supset_new(pXest_type,ptr_pXest, ptr_pXest_conn, old_comm, new_comm) @assert GridapDistributed.i_am_in(old_comm) pXest = ptr_pXest[] pXest_conn = ptr_pXest_conn[] pertree = Vector{P4est_wrapper.p4est_gloidx_t}(undef,pXest_conn.num_trees+1) - pXest_comm_count_pertree(Val{Dc},ptr_pXest,pertree) + pXest_comm_count_pertree(pXest_type,ptr_pXest,pertree) if (GridapDistributed.i_am_in(new_comm)) new_comm_num_parts = GridapDistributed.num_parts(new_comm) @@ -1539,9 +1465,9 @@ function _pXest_to_new_comm_old_supset_new(::Type{Val{Dc}},ptr_pXest, ptr_pXest_ for i = 1:length(new_global_first_quadrant) global_first_quadrant[i] = old_global_first_quadrant[i] end - quadrants = pXest_deflate_quadrants(Val{Dc},ptr_pXest,C_NULL) + quadrants = pXest_deflate_quadrants(pXest_type,ptr_pXest,C_NULL) - return pXest_inflate(Val{Dc}, + return pXest_inflate(pXest_type, new_comm, ptr_pXest_conn, global_first_quadrant, @@ -1554,76 +1480,6 @@ function _pXest_to_new_comm_old_supset_new(::Type{Val{Dc}},ptr_pXest, ptr_pXest_ end end -function _pXest_comm_find_owner(::Type{Val{Dc}},ptr_pXest,itree,quad,guess) where Dc - if (Dc==2) - return p4est_comm_find_owner(ptr_pXest,itree,quad,guess) - elseif (Dc==3) - return p8est_comm_find_owner(ptr_pXest,itree,quad,guess) - end -end - -function _p4est_compute_migration_control_data(::Type{Val{Dc}},ptr_pXest_old,ptr_pXest_new) where Dc - pXest_old = ptr_pXest_old[] - pXest_new = ptr_pXest_new[] - num_trees = Cint(pXest_old.connectivity[].num_trees) - my_rank = pXest_old.mpirank - ranks_count = Dict{Int,Int}() - lst_ranks = Int[] - old2new = Vector{Int}(undef,pXest_old.local_num_quadrants) - current_old_quad_index = 1 - - pXest_type = _dim_to_pXest_type(Dc) - - for itree = 0:num_trees-1 - tree = pXest_tree_array_index(pXest_type,pXest_old,itree)[] - num_quads = Cint(tree.quadrants.elem_count) - - for iquad = 0:num_quads-1 - q = pXest_quadrant_array_index(pXest_type, tree, iquad) - new_rank = _pXest_comm_find_owner(Val{Dc},ptr_pXest_new,itree,q,0) - if (new_rank != my_rank) - if (!(new_rank+1 in keys(ranks_count))) - push!(lst_ranks,new_rank+1) - ranks_count[new_rank+1] = 0 - end - ranks_count[new_rank+1] += 1 - old2new[current_old_quad_index] = 0 - else - current_new_quad_index = 1 - new_tree = pXest_tree_array_index(pXest_type,pXest_new,pXest_new.first_local_tree)[] - for t = pXest_new.first_local_tree:pXest_new.last_local_tree - new_tree = pXest_tree_array_index(pXest_type,pXest_new,t)[] - if t == itree - break - end - current_new_quad_index += Cint(new_tree.quadrants.elem_count) - end - found = false - num_quads_new = Cint(new_tree.quadrants.elem_count) - for iquad_new = 0:num_quads_new-1 - q_new = pXest_quadrant_array_index(pXest_type, new_tree, iquad_new) - found = pXest_quadrant_is_equal(pXest_type,q,q_new) - if found - break - end - current_new_quad_index += 1 - end - Gridap.Helpers.@check found - old2new[current_old_quad_index] = current_new_quad_index - end - current_old_quad_index += 1 - end - end - - local_ids = [i for i=1:length(old2new) if old2new[i]==0] - ptr_ranks = Vector{Int32}(undef,length(ranks_count)+1) - ptr_ranks[1] = 1 - for (i,rank) in enumerate(lst_ranks) - ptr_ranks[i+1]=ptr_ranks[i]+ranks_count[rank] - end - - lst_ranks,PartitionedArrays.JaggedArray(local_ids,ptr_ranks),old2new -end function is_included(partsA,partsB) @assert GridapDistributed.i_am_in(partsA.comm) || GridapDistributed.i_am_in(partsB.comm) @@ -1683,7 +1539,7 @@ function _redistribute_parts_subseteq_parts_redistributed(model::OctreeDistribut if (parts_redistributed_model === model.parts) ptr_pXest_old = model.ptr_pXest else - ptr_pXest_old = _pXest_to_new_comm(Val{Dc}, + ptr_pXest_old = _pXest_to_new_comm(model.pXest_type, model.ptr_pXest, model.ptr_pXest_connectivity, model.parts.comm, @@ -1693,8 +1549,8 @@ function _redistribute_parts_subseteq_parts_redistributed(model::OctreeDistribut pXest_partition!(model.pXest_type, ptr_pXest_new) # Compute RedistributeGlue - parts_snd, lids_snd, old2new = _p4est_compute_migration_control_data(Val{Dc},ptr_pXest_old,ptr_pXest_new) - parts_rcv, lids_rcv, new2old = _p4est_compute_migration_control_data(Val{Dc},ptr_pXest_new,ptr_pXest_old) + parts_snd, lids_snd, old2new = pXest_compute_migration_control_data(model.pXest_type,ptr_pXest_old,ptr_pXest_new) + parts_rcv, lids_rcv, new2old = pXest_compute_migration_control_data(model.pXest_type,ptr_pXest_new,ptr_pXest_old) lids_rcv, parts_rcv, lids_snd, parts_snd, old2new, new2old = _to_pdata(parts, lids_rcv, parts_rcv, lids_snd, parts_snd, old2new, new2old) @@ -1702,8 +1558,8 @@ function _redistribute_parts_subseteq_parts_redistributed(model::OctreeDistribut glue = GridapDistributed.RedistributeGlue(parts,model.parts,parts_rcv,parts_snd,lids_rcv,lids_snd,old2new,new2old) # Extract ghost and lnodes - ptr_pXest_ghost = setup_pXest_ghost(Val{Dc}, ptr_pXest_new) - ptr_pXest_lnodes = setup_pXest_lnodes_nonconforming(Val{Dc}, ptr_pXest_new, ptr_pXest_ghost) + ptr_pXest_ghost = setup_pXest_ghost(model.pXest_type, ptr_pXest_new) + ptr_pXest_lnodes = setup_pXest_lnodes_nonconforming(model.pXest_type, ptr_pXest_new, ptr_pXest_ghost) # Build fine-grid mesh fmodel, non_conforming_glue = setup_non_conforming_distributed_discrete_model(model.pXest_type, @@ -1725,6 +1581,7 @@ function _redistribute_parts_subseteq_parts_redistributed(model::OctreeDistribut model.ptr_pXest_connectivity, ptr_pXest_new, model.pXest_type, + model.pXest_refinement_rule_type, false, model) return red_model, glue @@ -1761,13 +1618,13 @@ function _redistribute_parts_supset_parts_redistributed( # p4est_vtk_write_file(model.ptr_pXest, C_NULL, "model.ptr_pXest") ptr_pXest_old=pXest_copy(model.pXest_type,model.ptr_pXest) - pXest_partition_given!(Val{Dc}, ptr_pXest_old, num_cells_per_part) + pXest_partition_given!(model.pXest_type, ptr_pXest_old, num_cells_per_part) # p4est_vtk_write_file(ptr_pXest_old, C_NULL, "ptr_pXest_old") # ptr_pXest_old is distributed over supset_comm # once created, ptr_pXest_new is distributed over subset_comm - ptr_pXest_new = _pXest_to_new_comm(Val{Dc}, + ptr_pXest_new = _pXest_to_new_comm(model.pXest_type, ptr_pXest_old, model.ptr_pXest_connectivity, supset_comm, @@ -1775,9 +1632,9 @@ function _redistribute_parts_supset_parts_redistributed( # Compute RedistributeGlue parts_snd, lids_snd, old2new = - _p4est_compute_migration_control_data(Val{Dc},model.ptr_pXest,ptr_pXest_old) + pXest_compute_migration_control_data(model.pXest_type,model.ptr_pXest,ptr_pXest_old) parts_rcv, lids_rcv, new2old = - _p4est_compute_migration_control_data(Val{Dc},ptr_pXest_old,model.ptr_pXest) + pXest_compute_migration_control_data(model.pXest_type,ptr_pXest_old,model.ptr_pXest) pXest_destroy(model.pXest_type,ptr_pXest_old) @@ -1791,12 +1648,14 @@ function _redistribute_parts_supset_parts_redistributed( if (GridapDistributed.i_am_in(subset_comm)) # p4est_vtk_write_file(ptr_pXest_new, C_NULL, "ptr_pXest_new") + pXest_type = _dim_to_pXest_type(Dc) + # Extract ghost and lnodes - ptr_pXest_ghost = setup_pXest_ghost(Val{Dc}, ptr_pXest_new) - ptr_pXest_lnodes = setup_pXest_lnodes(Val{Dc}, ptr_pXest_new, ptr_pXest_ghost) + ptr_pXest_ghost = setup_pXest_ghost(pXest_type, ptr_pXest_new) + ptr_pXest_lnodes = setup_pXest_lnodes(pXest_type, ptr_pXest_new, ptr_pXest_ghost) # # Build fine-grid mesh - fmodel = setup_distributed_discrete_model(Val{Dc}, + fmodel = setup_distributed_discrete_model(pXest_type, parts_redistributed_model, model.coarse_model, model.ptr_pXest_connectivity, @@ -1804,7 +1663,6 @@ function _redistribute_parts_supset_parts_redistributed( ptr_pXest_ghost, ptr_pXest_lnodes) - pXest_type = _dim_to_pXest_type(Dc) pXest_lnodes_destroy(pXest_type,ptr_pXest_lnodes) pXest_ghost_destroy(pXest_type,ptr_pXest_ghost) @@ -1819,6 +1677,7 @@ function _redistribute_parts_supset_parts_redistributed( model.ptr_pXest_connectivity, ptr_pXest_new, pXest_type, + model.pXest_refinement_rule_type, false, model) return red_model, glue diff --git a/src/PXestTypeMethods.jl b/src/PXestTypeMethods.jl index c59bc5c..e119e0a 100644 --- a/src/PXestTypeMethods.jl +++ b/src/PXestTypeMethods.jl @@ -10,8 +10,6 @@ struct PXestUniformRefinementRuleType <: PXestRefinementRuleType end; struct PXestVerticalRefinementRuleType <: PXestRefinementRuleType end; struct PXestHorizontalRefinementRuleType <: PXestRefinementRuleType end; - - function pXest_destroy(pXest_type::P4estType, ptr_pXest) p4est_destroy(ptr_pXest) end @@ -250,6 +248,10 @@ function p6est_vertically_refine!(ptr_pXest, refine_fn_c, refine_replace_fn_c; i p6est_refine_layers_ext(ptr_pXest, Cint(0), Cint(-1), refine_fn_c, init_fn_c, refine_replace_fn_c) end +function p6est_vertically_coarsen!(ptr_pXest, coarsen_fn_c) + p6est_coarsen_layers(ptr_pXest, Cint(0), coarsen_fn_c, C_NULL) +end + function pXest_coarsen!(::P4estType, ptr_pXest, coarsen_fn_c) p4est_coarsen(ptr_pXest, Cint(0), coarsen_fn_c, C_NULL) end @@ -304,7 +306,7 @@ function pXest_partition!(::P4estType, ptr_pXest) end function pXest_partition!(::P6estType, ptr_pXest) - p6est_partition(ptr_pXest, 0, C_NULL) + p6est_partition(ptr_pXest, C_NULL) end function pXest_partition!(::P8estType, ptr_pXest) @@ -379,7 +381,7 @@ function P4EST_QUADRANT_LEN(l) end function p2est_quadrant_is_ancestor(a,b) - if (a[].level==b[].level) + if (a[].level>=b[].level) return false end return (b[].z >= a[].z && @@ -492,6 +494,40 @@ function p6est_vertically_refine_callbacks() refine_layer_fn_callback_c, refine_layer_replace_callback_c end +function p6est_vertically_coarsen_callbacks() + function coarsen_layer_callback(p6est::Ptr{p6est_t}, + which_tree::p4est_topidx_t, + column::Ptr{p4est_quadrant_t}, + layer::Ptr{Ptr{p2est_quadrant_t}}) + + num_children=2 + layers=unsafe_wrap(Array, layer, num_children) + + coarsen=Cint(1) + for quadrant_index=1:num_children + quadrant = layers[quadrant_index][] + # I have noticed that new quadrants created as by-product + # of the refininement process have quadrant.p.user_data == C_NULL + # Not sure why ... The following if-end takes care of this. + if (quadrant.p.user_data) == C_NULL + println("XXX: first if") + return Cint(0) + end + is_coarsen_flag=(unsafe_wrap(Array,Ptr{Cint}(quadrant.p.user_data),1)[])==coarsen_flag + if (!is_coarsen_flag) + println("XXX: second if") + return Cint(0) + end + end + println("XXX: function end") + return coarsen + end + coarsen_fn_callback_c = @cfunction($coarsen_layer_callback, + Cint, + (Ptr{p6est_t}, p4est_topidx_t, Ptr{p4est_quadrant_t}, Ptr{Ptr{p2est_quadrant_t}})) + + return coarsen_fn_callback_c +end function pXest_coarsen_callbacks(::P4estType) function coarsen_callback(forest_ptr::Ptr{p4est_t}, @@ -1033,7 +1069,9 @@ end function subface_to_hanging_edges_within_face(::P8estType) p8est_subface_to_hanging_edges_within_face -end +end + +const p6est_half_to_regular_vertices = [ 0 1; 2 3; 0 2; 1 3] function generate_cell_faces_and_non_conforming_glue(pXest_type::PXestType, ptr_pXest_lnodes, @@ -1047,7 +1085,6 @@ function generate_cell_faces_and_non_conforming_glue(pXest_type::PXestType, lnodes = ptr_pXest_lnodes[] element_nodes = unsafe_wrap(Array, lnodes.element_nodes, lnodes.vnodes * lnodes.num_local_elements) face_code = unsafe_wrap(Array, lnodes.face_code, lnodes.num_local_elements) - println(face_code) hanging_face = Vector{Cint}(undef, n_cell_faces) face_code_with_ghosts = map(partition(cell_prange)) do indices @assert length(face_code)==own_length(indices) @@ -1315,10 +1352,12 @@ function generate_cell_faces_and_non_conforming_glue(pXest_type::PXestType, # Process hanging vertex p4est_hanging_lvertex = pXest_face_corners[p4est_lface, hanging_vertex_lvertex_within_face+1] hanging_vertices_pairs_to_owner_face[(cell, PXEST_2_GRIDAP_VERTEX[p4est_hanging_lvertex+1])] = owner_face + + # Process hanging face + hanging_faces_pairs_to_owner_face[(cell, PXEST_2_GRIDAP_FACE[p4est_lface])] = (owner_face,half+1) else # Anisotropic refinement @assert half in 4:7 - p6est_half_to_regular_vertices = [ 0 1; 2 3; 0 2; 1 3] for regular_vertex_lvertex_within_face in p6est_half_to_regular_vertices[mod(half,4)+1,:] # Process regular vertex p4est_regular_lvertex = pXest_face_corners[p4est_lface, regular_vertex_lvertex_within_face+1] @@ -1331,11 +1370,14 @@ function generate_cell_faces_and_non_conforming_glue(pXest_type::PXestType, p4est_regular_lvertex + 1, p4est_gvertex, PXEST_2_GRIDAP_VERTEX) - end + end + subface = mod(half,2)+1 + + # Process hanging face + hanging_faces_pairs_to_owner_face[(cell, PXEST_2_GRIDAP_FACE[p4est_lface])] = (owner_face,subface) end - # Process hanging face - hanging_faces_pairs_to_owner_face[(cell, PXEST_2_GRIDAP_FACE[p4est_lface])] = (owner_face,half+1) + if (Dc==3) _subface_to_hanging_edges_within_subface = subface_to_hanging_edges_within_subface(pXest_type) @@ -1922,6 +1964,174 @@ function generate_grid_and_topology(pXest_type::PXestType, grid,topology end +function pXest_comm_find_owner(::P4estType,ptr_pXest,itree,quad,guess) + return p4est_comm_find_owner(ptr_pXest,itree,quad,guess) +end + +function pXest_comm_find_owner(::P8estType,ptr_pXest,itree,quad,guess) + return p8est_comm_find_owner(ptr_pXest,itree,quad,guess) +end + +function pXest_compute_migration_control_data(pXest_type::P4P8estType,ptr_pXest_old,ptr_pXest_new) + pXest_old = ptr_pXest_old[] + pXest_new = ptr_pXest_new[] + num_trees = Cint(pXest_old.connectivity[].num_trees) + my_rank = pXest_old.mpirank + ranks_count = Dict{Int,Int}() + lst_ranks = Int[] + old2new = Vector{Int}(undef,pXest_old.local_num_quadrants) + current_old_quad_index = 1 + + for itree = 0:num_trees-1 + tree = pXest_tree_array_index(pXest_type,pXest_old,itree)[] + num_quads = Cint(tree.quadrants.elem_count) + + for iquad = 0:num_quads-1 + q = pXest_quadrant_array_index(pXest_type, tree, iquad) + new_rank = pXest_comm_find_owner(pXest_type,ptr_pXest_new,itree,q,0) + if (new_rank != my_rank) + if (!(new_rank+1 in keys(ranks_count))) + push!(lst_ranks,new_rank+1) + ranks_count[new_rank+1] = 0 + end + ranks_count[new_rank+1] += 1 + old2new[current_old_quad_index] = 0 + else + current_new_quad_index = 1 + new_tree = pXest_tree_array_index(pXest_type,pXest_new,pXest_new.first_local_tree)[] + for t = pXest_new.first_local_tree:pXest_new.last_local_tree + new_tree = pXest_tree_array_index(pXest_type,pXest_new,t)[] + if t == itree + break + end + current_new_quad_index += Cint(new_tree.quadrants.elem_count) + end + found = false + num_quads_new = Cint(new_tree.quadrants.elem_count) + for iquad_new = 0:num_quads_new-1 + q_new = pXest_quadrant_array_index(pXest_type, new_tree, iquad_new) + found = pXest_quadrant_is_equal(pXest_type,q,q_new) + if found + break + end + current_new_quad_index += 1 + end + Gridap.Helpers.@check found + old2new[current_old_quad_index] = current_new_quad_index + end + current_old_quad_index += 1 + end + end + local_ids = [i for i=1:length(old2new) if old2new[i]==0] + ptr_ranks = Vector{Int32}(undef,length(ranks_count)+1) + ptr_ranks[1] = 1 + for (i,rank) in enumerate(lst_ranks) + ptr_ranks[i+1]=ptr_ranks[i]+ranks_count[rank] + end + lst_ranks,PartitionedArrays.JaggedArray(local_ids,ptr_ranks),old2new +end +function pXest_compute_migration_control_data(pXest_type::P6estType,ptr_pXest_old,ptr_pXest_new) + + pXest_old = ptr_pXest_old[] + pXest_new = ptr_pXest_new[] + + lst_ranks, columns_snd_lids, columns_old2new= + pXest_compute_migration_control_data(P4estType(), + pXest_old.columns, + pXest_new.columns) + + ptrs=Vector{Int32}(undef,length(columns_snd_lids.ptrs)) + lids=Int32[] + old2new=Int32[] + + current_col = 1 + current_old_cell = 1 + current_new_cell = 1 + current_rank = 1 + + num_trees = Cint(pXest_old.columns[].connectivity[].num_trees) + + # Go over trees + for itree = 0:num_trees-1 + tree = pXest_tree_array_index(pXest_type,pXest_old,itree)[] + num_quads = Cint(tree.quadrants.elem_count) + + # Go over columns of current tree + for iquad=0:num_quads-1 + q = pXest_quadrant_array_index(pXest_type,tree,iquad) + f,l=P6EST_COLUMN_GET_RANGE(q[]) + num_quads_in_column=l-f + if (columns_old2new[current_col]==0) + if (current_col==columns_snd_lids.data[columns_snd_lids.ptrs[current_rank+1]]) + current_rank=current_rank+1 + end + ptrs[current_rank+1]+=num_quads_in_column + for i=0:num_quads_in_column-1 + push!(lids,current_old_cell+i) + end + else + for i=1:num_quads_in_column + push!(old2new,current_new_cell) + current_new_cell+=1 + end + end + current_old_cell+=num_quads_in_column + current_col+=1 + end + end + Gridap.Arrays.length_to_ptrs!(ptrs) + lst_ranks,PartitionedArrays.JaggedArray(lids,ptrs),old2new +end + +function pXest_deflate_quadrants(::P4estType,ptr_pXest,data) + P4est_wrapper.p4est_deflate_quadrants(ptr_pXest,data) +end + +function pXest_deflate_quadrants(::P8estType,ptr_pXest,data) + P4est_wrapper.p8est_deflate_quadrants(ptr_pXest,data) +end + +function pXest_comm_count_pertree(::P4estType,ptr_pXest,pertree) + p4est_comm_count_pertree(ptr_pXest,pertree) +end + +function pXest_comm_count_pertree(::P8estType,ptr_pXest,pertree) + p8est_comm_count_pertree(ptr_pXest,pertree) +end + +function pXest_inflate(::P4estType, + comm, + ptr_pXest_conn, + global_first_quadrant, + pertree, + quadrants, + data, + user_pointer) + P4est_wrapper.p4est_inflate(comm, + ptr_pXest_conn, + global_first_quadrant, + pertree, + quadrants, + data, + user_pointer) +end + +function pXest_inflate(::P8estType, + comm, + ptr_pXest_conn, + global_first_quadrant, + pertree, + quadrants, + data, + user_pointer) + P4est_wrapper.p8est_inflate(comm, + ptr_pXest_conn, + global_first_quadrant, + pertree, + quadrants, + data, + user_pointer) +end \ No newline at end of file diff --git a/src/UniformlyRefinedForestOfOctreesDiscreteModels.jl b/src/UniformlyRefinedForestOfOctreesDiscreteModels.jl index e66789a..b33cc70 100644 --- a/src/UniformlyRefinedForestOfOctreesDiscreteModels.jl +++ b/src/UniformlyRefinedForestOfOctreesDiscreteModels.jl @@ -1,20 +1,3 @@ - -function pXest_destroy(::Type{Val{Dc}}, ptr_pXest) where Dc - if (Dc==2) - p4est_destroy(ptr_pXest) - else - p8est_destroy(ptr_pXest) - end -end - -function pXest_connectivity_destroy(::Type{Val{Dc}}, ptr_pXest_connectivity) where Dc - if (Dc==2) - p4est_connectivity_destroy(ptr_pXest_connectivity) - else - p8est_connectivity_destroy(ptr_pXest_connectivity) - end -end - const P4EST_2_GRIDAP_FACET_2D = [ 3, 4, 1, 2 ] const GRIDAP_2_P4EST_FACET_2D = [ 3, 4, 1, 2 ] @@ -214,96 +197,6 @@ function setup_pXest_connectivity( pconn end -function setup_pXest(::Type{Val{Dc}}, comm, connectivity, num_uniform_refinements) where Dc - if (Dc==2) - p4est_new_ext(comm, - connectivity, - Cint(0), Cint(num_uniform_refinements), Cint(1), Cint(0), - C_NULL, C_NULL) - else - p8est_new_ext(comm, - connectivity, - Cint(0), Cint(num_uniform_refinements), Cint(1), Cint(0), - C_NULL, C_NULL) - end -end - -function setup_pXest_ghost(::Type{Val{Dc}}, ptr_pXest) where Dc - if (Dc==2) - p4est_ghost_new(ptr_pXest,P4est_wrapper.P4EST_CONNECT_FULL) - else - p8est_ghost_new(ptr_pXest,P4est_wrapper.P8EST_CONNECT_FULL) - end -end - -function setup_cell_prange(::Type{Val{Dc}}, - parts::AbstractVector{<:Integer}, - ptr_pXest, - ptr_pXest_ghost) where Dc - comm = parts.comm - - pXest_ghost = ptr_pXest_ghost[] - pXest = ptr_pXest[] - - # Obtain ghost quadrants - if (Dc==2) - ptr_ghost_quadrants = Ptr{p4est_quadrant_t}(pXest_ghost.ghosts.array) - else - ptr_ghost_quadrants = Ptr{p8est_quadrant_t}(pXest_ghost.ghosts.array) - end - proc_offsets = unsafe_wrap(Array, pXest_ghost.proc_offsets, pXest_ghost.mpisize+1) - - global_first_quadrant = unsafe_wrap(Array, - pXest.global_first_quadrant, - pXest.mpisize+1) - - noids,firstgid,gho_to_glo,gho_to_own=map(parts) do part - gho_to_glo = Vector{Int}(undef, pXest_ghost.ghosts.elem_count) - gho_to_own = Vector{Int32}(undef, pXest_ghost.ghosts.elem_count) - k=1 - for i=1:pXest_ghost.mpisize - for j=proc_offsets[i]:proc_offsets[i+1]-1 - quadrant = ptr_ghost_quadrants[j+1] - piggy3 = quadrant.p.piggy3 - gho_to_glo[k] = global_first_quadrant[i]+piggy3.local_num+1 - gho_to_own[k] = Int32(i) - k=k+1 - end - end - pXest.local_num_quadrants,global_first_quadrant[part]+1,gho_to_glo,gho_to_own - end |> tuple_of_arrays - ngids = pXest.global_num_quadrants - - partition=map(parts,noids,firstgid,gho_to_glo,gho_to_own) do part, noids, firstgid, gho_to_glo, gho_to_own - owner = part - own_indices=OwnIndices(ngids,owner,(collect(firstgid:firstgid+noids-1))) - ghost_indices=GhostIndices(ngids,gho_to_glo,gho_to_own) - OwnAndGhostIndices(own_indices,ghost_indices) - end - # This is required to provide the hint that the communication - # pattern underlying partition is symmetric, so that we do not have - # to execute the algorithm the reconstructs the reciprocal in the - # communication graph - assembly_neighbors(partition;symmetric=true) - PRange(partition) -end - -function setup_pXest_lnodes(::Type{Val{Dc}}, ptr_pXest, ptr_pXest_ghost) where Dc - if (Dc==2) - p4est_lnodes_new(ptr_pXest, ptr_pXest_ghost, Cint(1)) - else - p8est_lnodes_new(ptr_pXest, ptr_pXest_ghost, Cint(1)) - end -end - -function setup_pXest_lnodes_nonconforming(::Type{Val{Dc}}, ptr_pXest, ptr_pXest_ghost) where Dc - if (Dc==2) - p4est_lnodes_new(ptr_pXest, ptr_pXest_ghost, Cint(-2)) - else - p8est_lnodes_new(ptr_pXest, ptr_pXest_ghost, Cint(-3)) - end -end - function fetch_vector_ghost_values_cache(vector_partition,partition) cache = PArrays.p_vector_cache(vector_partition,partition) map(reverse,cache) @@ -706,9 +599,6 @@ function generate_face_labeling(pXest_type::P4P8estType, # end end - - - update_face_to_entity_with_ghost_data!(facet_to_entity, cell_prange, num_faces(polytope,Dc-1), @@ -792,7 +682,6 @@ end function update_face_to_entity_with_ghost_data!( face_to_entity,cell_prange,num_faces_x_cell,cell_to_faces) - part_to_cell_to_entity = map(init_cell_to_face_entity, map(x->num_faces_x_cell,partition(cell_prange)), cell_to_faces, @@ -806,34 +695,33 @@ function update_face_to_entity_with_ghost_data!( part_to_cell_to_entity) end -function setup_ptr_pXest_objects(::Type{Val{Dc}}, +function setup_ptr_pXest_objects(pXest_type::PXestType, comm, coarse_discrete_model, - num_uniform_refinements) where Dc + num_uniform_refinements) ptr_pXest_connectivity=setup_pXest_connectivity(coarse_discrete_model) # Create a new forest - ptr_pXest = setup_pXest(Val{Dc},comm,ptr_pXest_connectivity,num_uniform_refinements) + ptr_pXest = setup_pXest(pXest_type,comm,ptr_pXest_connectivity,num_uniform_refinements) # Build the ghost layer - ptr_pXest_ghost=setup_pXest_ghost(Val{Dc},ptr_pXest) - ptr_pXest_lnodes=setup_pXest_lnodes(Val{Dc}, ptr_pXest, ptr_pXest_ghost) + ptr_pXest_ghost=setup_pXest_ghost(pXest_type,ptr_pXest) + ptr_pXest_lnodes=setup_pXest_lnodes(pXest_type, ptr_pXest, ptr_pXest_ghost) ptr_pXest_connectivity, ptr_pXest, ptr_pXest_ghost, ptr_pXest_lnodes end -function setup_distributed_discrete_model(::Type{Val{Dc}}, +function setup_distributed_discrete_model(pXest_type::PXestType, parts, coarse_discrete_model, ptr_pXest_connectivity, ptr_pXest, ptr_pXest_ghost, - ptr_pXest_lnodes) where Dc + ptr_pXest_lnodes) - cell_prange = setup_cell_prange(Val{Dc},parts,ptr_pXest,ptr_pXest_ghost) + cell_prange = setup_cell_prange(pXest_type,parts,ptr_pXest,ptr_pXest_ghost) cell_vertex_gids=generate_cell_vertex_gids(ptr_pXest_lnodes,cell_prange) cell_vertex_lids,nlvertices=generate_cell_vertex_lids_nlvertices(cell_vertex_gids) - pXest_type = _dim_to_pXest_type(Dc) node_coordinates=generate_node_coordinates(pXest_type, cell_vertex_lids, @@ -873,14 +761,14 @@ function UniformlyRefinedForestOfOctreesDiscreteModel( ptr_pXest_connectivity, ptr_pXest, ptr_pXest_ghost, - ptr_pXest_lnodes = setup_ptr_pXest_objects(Val{Dc}, + ptr_pXest_lnodes = setup_ptr_pXest_objects(pXest_type, comm, coarse_discrete_model, num_uniform_refinements) # Write forest to VTK file # p4est_vtk_write_file(unitsquare_forest, C_NULL, "my_step") - dmodel=setup_distributed_discrete_model(Val{Dc}, + dmodel=setup_distributed_discrete_model(pXest_type, parts, coarse_discrete_model, ptr_pXest_connectivity, @@ -888,9 +776,9 @@ function UniformlyRefinedForestOfOctreesDiscreteModel( ptr_pXest_ghost, ptr_pXest_lnodes) - pXest_lnodes_destroy(Val{Dc},ptr_pXest_lnodes) - pXest_ghost_destroy(Val{Dc},ptr_pXest_ghost) - pXest_destroy(Val{Dc},ptr_pXest) - pXest_connectivity_destroy(Val{Dc},ptr_pXest_connectivity) + pXest_lnodes_destroy(pXest_type,ptr_pXest_lnodes) + pXest_ghost_destroy(pXest_type,ptr_pXest_ghost) + pXest_destroy(pXest_type,ptr_pXest) + pXest_connectivity_destroy(pXest_type,ptr_pXest_connectivity) dmodel end diff --git a/test.jl b/test.jl deleted file mode 100644 index 5c11c4f..0000000 --- a/test.jl +++ /dev/null @@ -1,33 +0,0 @@ -using Gridap -using GridapP4est -using MPI -using PartitionedArrays -using GridapDistributed -using Logging - -debug_logger = ConsoleLogger(stderr, Logging.Debug) -global_logger(debug_logger); # Enable the debug logger globally - -MPI.Init() - -ranks = with_mpi() do distribute - distribute(LinearIndices((prod(MPI.Comm_size(MPI.COMM_WORLD)),))) -end - -coarse_model=CartesianDiscreteModel((0,1,0,1),(1,1)) - -model=AnisotropicallyAdapted3DDistributedDiscreteModel(ranks,coarse_model,1,1) - -writevtk(model, "model"); - -ref_coarse_flags=map(ranks,partition(get_cell_gids(model.dmodel))) do rank,indices - flags=zeros(Int,length(indices)) - flags.=nothing_flag - flags[1]=refine_flag - flags[end]=refine_flag - flags -end - -model,glue=GridapP4est.vertically_adapt(model,ref_coarse_flags); - -writevtk(Triangulation(model), "trian"); \ No newline at end of file diff --git a/test/PoissonAnisotropicOctreeModelsTests.jl b/test/PoissonAnisotropicOctreeModelsTests.jl new file mode 100644 index 0000000..b952b7b --- /dev/null +++ b/test/PoissonAnisotropicOctreeModelsTests.jl @@ -0,0 +1,271 @@ +module PoissonAnisotropicOctreeModelsTests + using P4est_wrapper + using GridapP4est + using Gridap + using PartitionedArrays + using GridapDistributed + using MPI + using Gridap.FESpaces + using FillArrays + using Logging + using LinearAlgebra + + include("CoarseDiscreteModelsTools.jl") + + function generate_analytical_problem_functions(T::Type{Float64},order) + # Define manufactured functions + u(x) = x[1]+x[2]^order + f(x) = -Δ(u)(x) + u,f + end + + function generate_analytical_problem_functions(T::Type{VectorValue{3,Float64}},order) + # Define manufactured functions + u(x) = VectorValue(x[1]+x[2]^order+x[3],x[1]^order+x[2]+x[3],x[1]+x[2]+x[3]^order) + f(x) = -Δ(u)(x) + u,f + end + + function test_transfer_ops_and_redistribute(ranks,dmodel,order,T::Type) + # Define manufactured functions + u,f = generate_analytical_problem_functions(T,order) + degree = 2*order+1 + reffe=ReferenceFE(lagrangian,T,order) + VH=FESpace(dmodel,reffe;dirichlet_tags="boundary") + UH=TrialFESpace(VH,u) + ref_coarse_flags=map(ranks,partition(get_cell_gids(dmodel.dmodel))) do rank,indices + flags=zeros(Cint,length(indices)) + flags.=nothing_flag + + flags[1]=refine_flag + flags[own_length(indices)]=refine_flag + + # To create some unbalance + if (rank%2==0 && own_length(indices)>1) + flags[div(own_length(indices),2)]=refine_flag + end + flags + end + fmodel,glue=GridapP4est.vertically_adapt(dmodel,ref_coarse_flags); + + Vh=FESpace(fmodel,reffe,conformity=:H1;dirichlet_tags="boundary") + Uh=TrialFESpace(Vh,u) + + ΩH = Triangulation(dmodel) + dΩH = Measure(ΩH,degree) + + aH(u,v) = ∫( ∇(v)⊙∇(u) )*dΩH + bH(v) = ∫(v⋅f)*dΩH + + op = AffineFEOperator(aH,bH,UH,VH) + uH = solve(op) + e = u - uH + + # # Compute errors + el2 = sqrt(sum( ∫( e⋅e )*dΩH )) + eh1 = sqrt(sum( ∫( e⋅e + ∇(e)⊙∇(e) )*dΩH )) + + tol=1e-5 + println("[SOLVE COARSE] el2 < tol: $(el2) < $(tol)") + println("[SOLVE COARSE] eh1 < tol: $(eh1) < $(tol)") + @assert el2 < tol + @assert eh1 < tol + + Ωh = Triangulation(fmodel) + dΩh = Measure(Ωh,degree) + + ah(u,v) = ∫( ∇(v)⊙∇(u) )*dΩh + bh(v) = ∫(v⋅f)*dΩh + + op = AffineFEOperator(ah,bh,Uh,Vh) + uh = solve(op) + + # println(op.op.matrix.matrix_partition.item_ref[]*uh2dofs.vector_partition.item_ref[]- + # op.op.vector.vector_partition.item_ref[]) + #writevtk(ΩH, "ctrian", cellfields=["uH"=>uH]) + #writevtk(Ωh, "ftrian", cellfields=["uh"=>uh]) + + # # Compute errors + e = u - uh + el2 = sqrt(sum( ∫( e⋅e )*dΩh )) + eh1 = sqrt(sum( ∫( e⋅e + ∇(e)⊙∇(e) )*dΩh )) + + println("[SOLVE FINE] el2 < tol: $(el2) < $(tol)") + println("[SOLVE FINE] eh1 < tol: $(eh1) < $(tol)") + @assert el2 < tol + @assert eh1 < tol + + # prolongation via interpolation + uHh=interpolate(uH,Uh) + e = uh - uHh + el2 = sqrt(sum( ∫( e⋅e )*dΩh )) + println("[INTERPOLATION] el2 < tol: $(el2) < $(tol)") + @assert el2 < tol + + # prolongation via L2-projection + # Coarse FEFunction -> Fine FEFunction, by projection + ahp(u,v) = ∫(v⋅u)*dΩh + lhp(v) = ∫(v⋅uH)*dΩh + oph = AffineFEOperator(ahp,lhp,Uh,Vh) + uHh = solve(oph) + e = uh - uHh + el2 = sqrt(sum( ∫( e⋅e )*dΩh )) + println("[L2 PROJECTION] el2 < tol: $(el2) < $(tol)") + @assert el2 < tol + + # restriction via interpolation + uhH=interpolate(uh,UH) + e = uH - uhH + el2 = sqrt(sum( ∫( e⋅e )*dΩh )) + println("[INTERPOLATION] el2 < tol: $(el2) < $(tol)") + @assert el2 < tol + + # restriction via L2-projection + dΩhH = Measure(ΩH,Ωh,2*order) + aHp(u,v) = ∫(v⋅u)*dΩH + lHp(v) = ∫(v⋅uh)*dΩhH + oph = AffineFEOperator(aHp,lHp,UH,VH) + uhH = solve(oph) + e = uH - uhH + el2 = sqrt(sum( ∫( e⋅e )*dΩH )) + + fmodel_red, red_glue=GridapDistributed.redistribute(fmodel); + Vhred=FESpace(fmodel_red,reffe,conformity=:H1;dirichlet_tags="boundary") + Uhred=TrialFESpace(Vhred,u) + + Ωhred = Triangulation(fmodel_red) + dΩhred = Measure(Ωhred,degree) + + ahred(u,v) = ∫( ∇(v)⊙∇(u) )*dΩhred + bhred(v) = ∫(v⋅f)*dΩhred + + op = AffineFEOperator(ahred,bhred,Uhred,Vhred) + uhred = solve(op) + e = u - uhred + el2 = sqrt(sum( ∫( e⋅e )*dΩhred )) + println("[SOLVE FINE REDISTRIBUTED] el2 < tol: $(el2) < $(tol)") + @assert el2 < tol + + + uhred2 = GridapDistributed.redistribute_fe_function(uh,Vhred,fmodel_red,red_glue) + e = u - uhred2 + el2 = sqrt(sum( ∫( e⋅e )*dΩhred )) + println("[REDISTRIBUTE SOLUTION] el2 < tol: $(el2) < $(tol)") + @assert el2 < tol + + fmodel_red + end + + function test_refine_and_coarsen_at_once(ranks, + dmodel::OctreeDistributedDiscreteModel{Dc}, + order, + T::Type) where Dc + + # Define manufactured functions + u,f = generate_analytical_problem_functions(T,order) + + degree = 2*order+1 + ref_coarse_flags=map(ranks,partition(get_cell_gids(dmodel.dmodel))) do rank,indices + flags=zeros(Int,length(indices)) + flags.=nothing_flag + if (rank==1) + flags[1:2].=coarsen_flag + else + flags[own_length(indices)]=refine_flag + end + flags + end + fmodel,glue=GridapP4est.vertically_adapt(dmodel,ref_coarse_flags); + + reffe=ReferenceFE(lagrangian,T,order) + VH=FESpace(dmodel,reffe,conformity=:H1;dirichlet_tags="boundary") + UH=TrialFESpace(VH,u) + + Vh=FESpace(fmodel,reffe,conformity=:H1;dirichlet_tags="boundary") + Uh=TrialFESpace(Vh,u) + ΩH = Triangulation(dmodel) + dΩH = Measure(ΩH,degree) + + aH(u,v) = ∫( ∇(v)⊙∇(u) )*dΩH + bH(v) = ∫(v⋅f)*dΩH + + op = AffineFEOperator(aH,bH,UH,VH) + uH = solve(op) + e = u - uH + + # # Compute errors + el2 = sqrt(sum( ∫( e⋅e )*dΩH )) + eh1 = sqrt(sum( ∫( e⋅e + ∇(e)⊙∇(e) )*dΩH )) + + tol=1e-5 + println("[SOLVE INITIAL MESH] el2 < tol: $(el2) < $(tol)") + println("[SOLVE INITIAL MESH] eh1 < tol: $(eh1) < $(tol)") + @assert el2 < tol + @assert eh1 < tol + + Ωh = Triangulation(fmodel) + dΩh = Measure(Ωh,degree) + + ah(u,v) = ∫( ∇(v)⊙∇(u) )*dΩh + bh(v) = ∫(v⋅f)*dΩh + + op = AffineFEOperator(ah,bh,Uh,Vh) + uh = solve(op) + e = u - uh + + # # Compute errors + el2 = sqrt(sum( ∫( e⋅e )*dΩh )) + eh1 = sqrt(sum( ∫( e⋅e + ∇(e)⊙∇(e) )*dΩh )) + println("[SOLVE ADAPTED MESH] el2 < tol: $(el2) < $(tol)") + println("[SOLVE ADAPTED MESH] eh1 < tol: $(eh1) < $(tol)") + @assert el2 < tol + @assert eh1 < tol + + # prolongation via interpolation + uHh=interpolate(uH,Uh) + e = uh - uHh + el2 = sqrt(sum( ∫( e⋅e )*dΩh )) + println("[INTERPOLATION INITIAL-ADAPTED MESH] el2 < tol: $(el2) < $(tol)") + @assert el2 < tol + end + + function test_3d(ranks,order,T::Type;num_amr_steps=5) + coarse_model=CartesianDiscreteModel((0,1,0,1),(1,1)) + dmodel=AnisotropicallyAdapted3DDistributedDiscreteModel(ranks, coarse_model, 1, 1) + test_refine_and_coarsen_at_once(ranks,dmodel,order,T) + rdmodel=dmodel + for i=1:num_amr_steps + rdmodel=test_transfer_ops_and_redistribute(ranks,rdmodel,order,T) + end + end + + function test(ranks,perm, order,T::Type) + coarse_model = setup_model(Val{2},perm) + model = AnisotropicallyAdapted3DDistributedDiscreteModel(ranks, coarse_model, 1, 1) + test_transfer_ops_and_redistribute(ranks,model,order,T) + end + + function _field_type(::Val{Dc}, scalar_or_vector::Symbol) where Dc + if scalar_or_vector==:scalar + Float64 + else + @assert scalar_or_vector==:vector + VectorValue{Dc,Float64} + end + end + function run(distribute) + #debug_logger = ConsoleLogger(stderr, Logging.Debug) + #global_logger(debug_logger); # Enable the debug logger globally + ranks = distribute(LinearIndices((MPI.Comm_size(MPI.COMM_WORLD),))) + for perm=1:4, order=1:4, scalar_or_vector in (:scalar,) + test(ranks,perm,order,_field_type(Val{3}(),scalar_or_vector)) + end + for perm in (1,2), order in (1,4), scalar_or_vector in (:vector,) + test(ranks,perm,order,_field_type(Val{3}(),scalar_or_vector)) + end + for order=2:2, scalar_or_vector in (:scalar,:vector) + test_3d(ranks,order,_field_type(Val{3}(),scalar_or_vector), num_amr_steps=4) + end + end +end + diff --git a/test/mpi/PoissonAnisotropicOctreeModelsTests.jl b/test/mpi/PoissonAnisotropicOctreeModelsTests.jl new file mode 100644 index 0000000..042a93e --- /dev/null +++ b/test/mpi/PoissonAnisotropicOctreeModelsTests.jl @@ -0,0 +1,17 @@ + +using MPI +using PartitionedArrays + +include("../PoissonAnisotropicOctreeModelsTests.jl") +import .PoissonAnisotropicOctreeModelsTests as TestModule + +if !MPI.Initialized() + MPI.Init() +end + +with_mpi() do distribute + TestModule.run(distribute) +end + +MPI.Finalize() + From 8fc120755261f9511ac291d7d83eeb0066450d03 Mon Sep 17 00:00:00 2001 From: "Alberto F. Martin" Date: Sun, 3 Mar 2024 19:38:20 +1100 Subject: [PATCH 03/29] Horizontal refinement working in serial --- Project.toml | 2 +- ...callyAdapted3DDistributedDiscreteModels.jl | 210 +++++++- src/FESpaces.jl | 236 +++++---- src/GridapP4est.jl | 6 +- src/OctreeDistributedDiscreteModels.jl | 78 ++- src/PXestTypeMethods.jl | 475 ++++++++++++++---- ...mlyRefinedForestOfOctreesDiscreteModels.jl | 11 +- test/PoissonAnisotropicOctreeModelsTests.jl | 66 +-- test/PoissonNonConformingOctreeModelsTests.jl | 4 +- 9 files changed, 826 insertions(+), 262 deletions(-) diff --git a/Project.toml b/Project.toml index 12cf8ec..800011d 100644 --- a/Project.toml +++ b/Project.toml @@ -19,7 +19,7 @@ ArgParse = "1" FillArrays = "0.8.4, 0.9, 0.10, 0.11, 0.12, 1" Gridap = "0.17.22" MPI = "0.20" -P4est_wrapper = "0.2.0" +P4est_wrapper = "0.2.1" PartitionedArrays = "0.3.3" julia = "1.5,1.6,1.7,1.8,1.9" diff --git a/src/AnisotropicallyAdapted3DDistributedDiscreteModels.jl b/src/AnisotropicallyAdapted3DDistributedDiscreteModels.jl index 25a2064..35a9e21 100644 --- a/src/AnisotropicallyAdapted3DDistributedDiscreteModels.jl +++ b/src/AnisotropicallyAdapted3DDistributedDiscreteModels.jl @@ -97,12 +97,14 @@ function vertically_adapt(model::OctreeDistributedDiscreteModel{Dc,Dp}, pXest_ghost_destroy(model.pXest_type,ptr_pXest_ghost) pXest_lnodes_destroy(model.pXest_type,ptr_pXest_lnodes) pXest_refinement_rule_type = PXestVerticalRefinementRuleType() + stride = pXest_stride_among_children(model.pXest_type,pXest_refinement_rule_type,model.ptr_pXest) adaptivity_glue = _compute_fine_to_coarse_model_glue(model.pXest_type, pXest_refinement_rule_type, model.parts, model.dmodel, fmodel, - _refinement_and_coarsening_flags) + _refinement_and_coarsening_flags, + stride) adaptive_models = map(local_views(model), local_views(fmodel), adaptivity_glue) do model, fmodel, glue @@ -403,11 +405,27 @@ function generate_face_labeling(pXest_type::P6estType, facet_to_entity = map(x->x[Dc] , faces_to_entity) cell_to_entity = map(x->x[Dc+1], faces_to_entity) - function cell_to_faces(topology,cell_dim,face_dim) - map(topology) do topology - Gridap.Geometry.get_faces(topology,cell_dim,face_dim) - end - end + polytope = HEX + + update_face_to_entity_with_ghost_data!(vertex_to_entity, + cell_prange, + num_faces(polytope,0), + cell_to_faces(topology,Dc,0)) + + update_face_to_entity_with_ghost_data!(edget_to_entity, + cell_prange, + num_faces(polytope,1), + cell_to_faces(topology,Dc,1)) + + update_face_to_entity_with_ghost_data!(facet_to_entity, + cell_prange, + num_faces(polytope,Dc-1), + cell_to_faces(topology,Dc,Dc-1)) + + update_face_to_entity_with_ghost_data!(cell_to_entity, + cell_prange, + num_faces(polytope,Dc), + cell_to_faces(topology,Dc,Dc)) faces_to_entity=[vertex_to_entity,edget_to_entity,facet_to_entity,cell_to_entity] @@ -460,3 +478,183 @@ face_labeling = end face_labeling end + +function num_locally_owned_columns(octree_model) + @assert octree_model.pXest_type==P6estType() + map(octree_model.parts) do _ + pXest=octree_model.ptr_pXest[] + num_cols = 0 + num_trees = Cint(pXest.columns[].connectivity[].num_trees) + for itree = 0:num_trees-1 + tree = pXest_tree_array_index(octree_model.pXest_type,pXest,itree)[] + num_cols += tree.quadrants.elem_count + end + num_cols + end +end + + +function _horizontally_refine_coarsen_balance!(model::OctreeDistributedDiscreteModel{Dc,Dp}, + refinement_and_coarsening_flags::MPIArray{<:Vector}) where {Dc,Dp} + + pXest_type = model.pXest_type + init_fn_callback_c = p6est_horizontally_adapt_reset_callbacks() + coarsen_fn_callback_c = p6est_horizontally_coarsen_callbacks() + refine_callback_c,refine_replace_callback_c = p6est_horizontally_refine_callbacks() + + num_cols = num_locally_owned_columns(model) + + map(refinement_and_coarsening_flags,num_cols) do flags, num_cols + # The length of the local flags array has to match the number of locally owned columns in the model + @assert num_cols==length(flags) + println("BBB: $(flags)") + pXest_reset_data!(pXest_type, model.ptr_pXest, Cint(sizeof(Cint)), init_fn_callback_c, pointer(flags)) + println("CCC: $(flags)") + end + + + # # Copy input p4est, refine and balance + ptr_new_pXest = pXest_copy(pXest_type, model.ptr_pXest) + + p6est_horizontally_refine!(ptr_new_pXest, + refine_callback_c, + refine_replace_callback_c) + + p6est_horizontally_coarsen!(ptr_new_pXest, coarsen_fn_callback_c) + + pXest_balance!(pXest_type, ptr_new_pXest) + + p6est_horizontally_adapt_update_flags!(model.ptr_pXest,ptr_new_pXest) + + ptr_new_pXest +end + + +function horizontally_adapt(model::OctreeDistributedDiscreteModel{Dc,Dp}, + refinement_and_coarsening_flags::MPIArray{<:Vector{<:Integer}}; + parts=nothing) where {Dc,Dp} + + Gridap.Helpers.@notimplementedif parts!=nothing + + _refinement_and_coarsening_flags = map(refinement_and_coarsening_flags) do flags + convert(Vector{Cint},flags) + end + + ptr_new_pXest = _horizontally_refine_coarsen_balance!(model, _refinement_and_coarsening_flags) + + # Extract ghost and lnodes + ptr_pXest_ghost = setup_pXest_ghost(model.pXest_type, ptr_new_pXest) + ptr_pXest_lnodes = setup_pXest_lnodes_nonconforming(model.pXest_type, ptr_new_pXest, ptr_pXest_ghost) + + # Build fine-grid mesh + fmodel,non_conforming_glue = setup_non_conforming_distributed_discrete_model(model.pXest_type, + model.parts, + model.coarse_model, + model.ptr_pXest_connectivity, + ptr_new_pXest, + ptr_pXest_ghost, + ptr_pXest_lnodes) + + pXest_ghost_destroy(model.pXest_type,ptr_pXest_ghost) + pXest_lnodes_destroy(model.pXest_type,ptr_pXest_lnodes) + pXest_refinement_rule_type = PXestHorizontalRefinementRuleType() + + _extruded_flags = _extrude_refinement_and_coarsening_flags(_refinement_and_coarsening_flags, + model.ptr_pXest, + ptr_new_pXest) + stride = pXest_stride_among_children(model.pXest_type,pXest_refinement_rule_type,model.ptr_pXest) + adaptivity_glue = _compute_fine_to_coarse_model_glue(model.pXest_type, + pXest_refinement_rule_type, + model.parts, + model.dmodel, + fmodel, + _extruded_flags, + stride) + adaptive_models = map(local_views(model), + local_views(fmodel), + adaptivity_glue) do model, fmodel, glue + Gridap.Adaptivity.AdaptedDiscreteModel(fmodel,model,glue) + end + fmodel = GridapDistributed.GenericDistributedDiscreteModel(adaptive_models,get_cell_gids(fmodel)) + ref_model = OctreeDistributedDiscreteModel(Dc,Dp, + model.parts, + fmodel, + non_conforming_glue, + model.coarse_model, + model.ptr_pXest_connectivity, + ptr_new_pXest, + model.pXest_type, + pXest_refinement_rule_type, + false, + model) + return ref_model, adaptivity_glue +end + + function _extrude_refinement_and_coarsening_flags( + refinement_and_coarsening_flags::MPIArray{<:Vector{<:Integer}}, + ptr_pXest_old, + ptr_pXest_new) + + pXest_old = ptr_pXest_old[] + pXest_new = ptr_pXest_new[] + pXest_type = P6estType() + + num_trees = Cint(pXest_old.columns[].connectivity[].num_trees) + @assert num_trees == Cint(pXest_new.columns[].connectivity[].num_trees) + + map(refinement_and_coarsening_flags) do flags + current_old_quad=1 + current_cell_old=1 + total=0 + for itree=0:num_trees-1 + tree = pXest_tree_array_index(pXest_type,pXest_old,itree)[] + num_quads = Cint(tree.quadrants.elem_count) + q = pXest_quadrant_array_index(pXest_type,tree,0) + f,l=P6EST_COLUMN_GET_RANGE(q[]) + num_layers=l-f + total+=num_quads*num_layers + end + extruded_flags=similar(flags, total) + + # Go over trees + for itree=0:num_trees-1 + tree = pXest_tree_array_index(pXest_type,pXest_old,itree)[] + num_quads = Cint(tree.quadrants.elem_count) + iquad=0 + # Go over columns of current tree + while iquad0) + max_negative_minus_one = -maximum(-U_cell_dof_ids.data) - 1 + else + max_negative_minus_one = 0 + end # max_negative_minus_one can only be zero whenever there are no # negative values in U_cell_dof_ids.data (i.e., no Dirichlet DoFs) if (max_negative_minus_one==0) diff --git a/src/GridapP4est.jl b/src/GridapP4est.jl index be542e7..a56bcca 100644 --- a/src/GridapP4est.jl +++ b/src/GridapP4est.jl @@ -24,10 +24,8 @@ module GridapP4est export UniformlyRefinedForestOfOctreesDiscreteModel export OctreeDistributedDiscreteModel export AnisotropicallyAdapted3DDistributedDiscreteModel - export adapt - export refine - export coarsen - export redistribute + export vertically_adapt + export horizontally_adapt export nothing_flag, refine_flag, coarsen_flag export FixedFractionAdaptiveFlagsMarkingStrategy export update_adaptivity_flags! diff --git a/src/OctreeDistributedDiscreteModels.jl b/src/OctreeDistributedDiscreteModels.jl index d8b3094..d49860c 100644 --- a/src/OctreeDistributedDiscreteModels.jl +++ b/src/OctreeDistributedDiscreteModels.jl @@ -290,16 +290,20 @@ end function pXest_update_flags!(pXest_type::P4P8estType, ptr_pXest_old, ptr_pXest_new) pXest_old = ptr_pXest_old[] - pXest_new = ptr_pXest_new[] flags=unsafe_wrap(Array, Ptr{Cint}(pXest_old.user_pointer), pXest_old.local_num_quadrants) + pXest_update_flags!(pXest_type, flags, ptr_pXest_old, ptr_pXest_new) +end + +function pXest_update_flags!(pXest_type::P4P8estType, flags, ptr_pXest_old, ptr_pXest_new) + pXest_old = ptr_pXest_old[] + pXest_new = ptr_pXest_new[] num_trees = Cint(pXest_old.connectivity[].num_trees) @assert num_trees == Cint(pXest_new.connectivity[].num_trees) Dc=num_cell_dims(pXest_type) - num_children = get_num_children(pXest_type, PXestUniformRefinementRuleType()) global_iquad_old = 0 for itree = 0:num_trees-1 @@ -335,6 +339,17 @@ function pXest_update_flags!(pXest_type::P4P8estType, ptr_pXest_old, ptr_pXest_n end end +function p6est_horizontally_adapt_update_flags!(ptr_pXest_old, ptr_pXest_new) + pXest_old = ptr_pXest_old[] + pXest_new = ptr_pXest_new[] + ptr_p4est_old = pXest_old.columns + ptr_p4est_new = pXest_new.columns + flags=unsafe_wrap(Array, + Ptr{Cint}(pXest_old.user_pointer), + pXest_old.columns[].local_num_quadrants) + pXest_update_flags!(P4estType(), flags, ptr_p4est_old, ptr_p4est_new) +end + function p6est_vertically_adapt_update_flags!(ptr_pXest_old, ptr_pXest_new) pXest_old = ptr_pXest_old[] pXest_new = ptr_pXest_new[] @@ -551,7 +566,8 @@ function _compute_fine_to_coarse_model_glue( cparts, cmodel::Union{Nothing,GridapDistributed.DistributedDiscreteModel{Dc}}, fmodel::GridapDistributed.DistributedDiscreteModel{Dc}, - refinement_and_coarsening_flags::MPIArray{<:AbstractVector}) where Dc + refinement_and_coarsening_flags::MPIArray{<:AbstractVector}, + stride) where Dc function setup_communication_buffers_fine_partition(cparts, fmodel, @@ -916,7 +932,8 @@ function _compute_fine_to_coarse_model_glue( cpartition, fpartition, flags, - coarsen) + coarsen, + stride) end end |> tuple_of_arrays @@ -1008,7 +1025,8 @@ function _process_owned_cells_fine_to_coarse_model_glue(pXest_type, cpartition, fpartition, flags, - coarsen) where Dc + coarsen, + stride) where Dc function _setup_fine_to_coarse_faces_map_table(pXest_type,pXest_refinement_rule_type,flags,num_o_c_cells,num_f_cells) num_children = get_num_children(pXest_type, pXest_refinement_rule_type) @@ -1095,21 +1113,34 @@ function _process_owned_cells_fine_to_coarse_model_glue(pXest_type, Gridap.Arrays.Table(fcell_to_child_id_data, fine_to_coarse_faces_map_ptrs) end - function _setup_fine_to_coarse_faces_map_vector!(pXest_type,pXest_refinement_rule_type,fine_to_coarse_faces_map,fcell_to_child_id,flags,num_o_c_cells) + function _setup_fine_to_coarse_faces_map_vector!(pXest_type, + pXest_refinement_rule_type, + fine_to_coarse_faces_map, + fcell_to_child_id, + flags, + num_o_c_cells, + stride) # Go over all cells of coarse grid portion num_children = get_num_children(pXest_type,pXest_refinement_rule_type) c = 1 - for cell = 1:num_o_c_cells + cell=1 + while cell<=num_o_c_cells if flags[cell]==refine_flag for child = 1:num_children - fine_to_coarse_faces_map[c+child-1] = cell - fcell_to_child_id[c+child-1] = child + current_cell=cell + for j=1:stride + fine_to_coarse_faces_map[c] = current_cell + fcell_to_child_id[c] = child + c+=1 + current_cell+=1 + end end - c = c + num_children + cell=cell+stride elseif (flags[cell]==nothing_flag) fine_to_coarse_faces_map[c] = cell fcell_to_child_id[c] = 1 - c=c+1 + c+=1 + cell+=1 else @assert flags[cell]!=coarsen_flag error("Unknown AMR flag") @@ -1131,7 +1162,12 @@ function _process_owned_cells_fine_to_coarse_model_glue(pXest_type, fine_to_coarse_faces_map = Vector{Vector{Int}}(undef,Dc+1) fine_to_coarse_faces_map[Dc+1] = Vector{Int}(undef,num_f_cells) fcell_to_child_id = Vector{Int}(undef,num_f_cells) - _setup_fine_to_coarse_faces_map_vector!(pXest_type,pXest_refinement_rule_type,fine_to_coarse_faces_map[Dc+1],fcell_to_child_id,flags,num_o_c_cells) + _setup_fine_to_coarse_faces_map_vector!(pXest_type, + pXest_refinement_rule_type, + fine_to_coarse_faces_map[Dc+1], + fcell_to_child_id,flags, + num_o_c_cells, + stride) for d=1:Dc fine_to_coarse_faces_map[d] = Vector{Int}(undef,Gridap.Geometry.num_faces(ftopology,d-1)) end @@ -1310,12 +1346,14 @@ function Gridap.Adaptivity.adapt(model::OctreeDistributedDiscreteModel{Dc,Dp}, pXest_ghost_destroy(model.pXest_type,ptr_pXest_ghost) pXest_lnodes_destroy(model.pXest_type,ptr_pXest_lnodes) pXest_refinement_rule_type = PXestUniformRefinementRuleType() + stride = pXest_stride_among_children(model.pXest_type,pXest_refinement_rule_type,model.ptr_pXest) adaptivity_glue = _compute_fine_to_coarse_model_glue(model.pXest_type, pXest_refinement_rule_type, model.parts, model.dmodel, fmodel, - _refinement_and_coarsening_flags) + _refinement_and_coarsening_flags, + stride) adaptive_models = map(local_views(model), local_views(fmodel), adaptivity_glue) do model, fmodel, glue @@ -1744,14 +1782,20 @@ const p8est_subface_to_hanging_edges_within_subface = 0 2; ] +# Here, we assign the identifier 5 to a "horizontal" hanging edge +# within a face, and the identifier 6 to a "vertical" hanging edge +# within a face const p6est_subface_to_hanging_edges_within_face = [ - 1; - 1; - 1; - 1; + 5; + 5; + 6; + 6; ] +# Here, we are assigning arbitrarily a local identifier +# within the master face to each hanging edge. +# 1: south, 2: north, 3: west and 4: east. const p8est_subface_to_hanging_edges_within_face = [ 3 1; diff --git a/src/PXestTypeMethods.jl b/src/PXestTypeMethods.jl index e119e0a..97ba91c 100644 --- a/src/PXestTypeMethods.jl +++ b/src/PXestTypeMethods.jl @@ -111,7 +111,7 @@ function setup_pXest_ghost(pXest_type::P6estType, ptr_pXest) end function setup_pXest_ghost(pXest_type::P8estType, ptr_pXest) - p8est_ghost_new(ptr_pXest,P4est_wrapper.P4EST_CONNECT_FULL) + p8est_ghost_new(ptr_pXest,P4est_wrapper.P8EST_CONNECT_FULL) end function setup_pXest_lnodes(pXest_type::P4estType, ptr_pXest, ptr_pXest_ghost) @@ -248,10 +248,18 @@ function p6est_vertically_refine!(ptr_pXest, refine_fn_c, refine_replace_fn_c; i p6est_refine_layers_ext(ptr_pXest, Cint(0), Cint(-1), refine_fn_c, init_fn_c, refine_replace_fn_c) end +function p6est_horizontally_refine!(ptr_pXest, refine_fn_c, refine_replace_fn_c; init_fn_c=C_NULL) + p6est_refine_columns_ext(ptr_pXest, Cint(0), Cint(-1), refine_fn_c, init_fn_c, refine_replace_fn_c) +end + function p6est_vertically_coarsen!(ptr_pXest, coarsen_fn_c) p6est_coarsen_layers(ptr_pXest, Cint(0), coarsen_fn_c, C_NULL) end +function p6est_horizontally_coarsen!(ptr_pXest, coarsen_fn_c) + p6est_coarsen_columns(ptr_pXest, Cint(0), coarsen_fn_c, C_NULL) +end + function pXest_coarsen!(::P4estType, ptr_pXest, coarsen_fn_c) p4est_coarsen(ptr_pXest, Cint(0), coarsen_fn_c, C_NULL) end @@ -435,8 +443,8 @@ function p6est_vertically_adapt_reset_callbacks() current_layer_within_column=current_layer_within_column+1 current_layer=current_layer+1 - if (which_tree==columns.connectivity[].num_trees && - current_quadrant_index_within_tree+1==tree.quadrants.elem_count && + if ((which_tree+1)==columns.connectivity[].num_trees && + current_quadrant_index_within_tree==tree.quadrants.elem_count && current_layer_within_column==l-f) current_quadrant_index_among_trees = Cint(-1) current_layer=Cint(0) @@ -450,6 +458,66 @@ function p6est_vertically_adapt_reset_callbacks() end + +function p6est_horizontally_adapt_reset_callbacks() + current_quadrant_index_among_trees = Cint(-1) + current_quadrant_index_within_tree = Cint(0) + current_layer_within_column = Cint(0) + current_layer = Cint(0) + previous_quadrant = Ref{p4est_quadrant_t}() + + + function init_fn_callback(forest_ptr::Ptr{p6est_t}, + which_tree::p4est_topidx_t, + column_ptr::Ptr{p4est_quadrant_t}, + layer_ptr::Ptr{p2est_quadrant_t}) + + forest = forest_ptr[] + columns = forest.columns[] + tree = p4est_tree_array_index(columns.trees, which_tree)[] + quadrant = column_ptr[] + layer = layer_ptr[] + + if (current_quadrant_index_among_trees==-1 || + p4est_quadrant_compare(previous_quadrant,column_ptr) != 0) + + previous_quadrant = column_ptr + current_quadrant_index_among_trees = current_quadrant_index_among_trees+1 + current_quadrant_index_within_tree = (current_quadrant_index_within_tree + 1) % (tree.quadrants.elem_count) + q = p4est_quadrant_array_index(tree.quadrants, current_quadrant_index_within_tree) + current_layer_within_column = 0 + end + + f,l=P6EST_COLUMN_GET_RANGE(column_ptr[]) + if (current_layer_within_column==0) + user_data = + unsafe_wrap(Array, Ptr{Cint}(forest.user_pointer), + current_quadrant_index_among_trees+1)[current_quadrant_index_among_trees+1] + # We will use the first layer of the first column + # to decide whether we refine the column or not + q2_ptr=p2est_quadrant_array_index(forest.layers[], f) + @assert p2est_quadrant_is_equal(q2_ptr,layer_ptr) + + unsafe_store!(Ptr{Cint}(q2_ptr[].p.user_data), user_data, 1) + end + + current_layer_within_column=current_layer_within_column+1 + current_layer=current_layer+1 + + if ((which_tree+1)==columns.connectivity[].num_trees && + current_quadrant_index_within_tree==tree.quadrants.elem_count && + current_layer_within_column==l-f) + current_quadrant_index_among_trees = Cint(-1) + current_layer=Cint(0) + current_quadrant_index_within_tree = Cint(0) + end + return nothing + end + @cfunction($init_fn_callback, Cvoid, + (Ptr{p6est_t}, p4est_topidx_t, Ptr{p4est_quadrant_t},Ptr{p2est_quadrant_t})) +end + + function p6est_vertically_refine_callbacks() function refine_layer_callback(p6est::Ptr{p6est_t}, which_tree::p4est_topidx_t, @@ -492,8 +560,53 @@ function p6est_vertically_refine_callbacks() Ptr{Ptr{p4est_quadrant_t}}, Ptr{Ptr{p2est_quadrant_t}})) refine_layer_fn_callback_c, refine_layer_replace_callback_c +end + +function p6est_horizontally_refine_callbacks() + function refine_column_callback(p6est::Ptr{p6est_t}, + which_tree::p4est_topidx_t, + column::Ptr{p4est_quadrant_t}) + forest=p6est[] + f,l=P6EST_COLUMN_GET_RANGE(column[]) + q2_ptr=p2est_quadrant_array_index(forest.layers[], f) + Cint(unsafe_wrap(Array, Ptr{Cint}(q2_ptr[].p.user_data), 1)[] == refine_flag) + end + + refine_column_fn_callback_c = @cfunction($refine_column_callback, Cint, + (Ptr{p6est_t}, p4est_topidx_t, Ptr{p4est_quadrant_t})) + + function refine_column_replace_callback(p6est::Ptr{p6est_t}, + which_tree::p4est_topidx_t, + num_outcolumns::Cint, + num_outlayers::Cint, + outcolumns::Ptr{Ptr{p4est_quadrant_t}}, + outlayers::Ptr{Ptr{p2est_quadrant_t}}, + num_incolumns::Cint, + num_inlayers::Cint, + incolumns::Ptr{Ptr{p4est_quadrant_t}}, + inlayers::Ptr{Ptr{p2est_quadrant_t}}) + + @assert num_outcolumns==1 + @assert num_incolumns==4 + + inlayers_array=unsafe_wrap(Array, inlayers, num_inlayers) + for i=1:num_inlayers + quadrant = inlayers_array[i][] + if (quadrant.p.user_data) != C_NULL + unsafe_store!(Ptr{Cint}(quadrant.p.user_data), nothing_flag, 1) + end + end + end + + refine_column_replace_callback_c = @cfunction($refine_column_replace_callback, Cvoid, + (Ptr{p6est_t}, p4est_topidx_t, Cint, Cint, + Ptr{Ptr{p4est_quadrant_t}}, Ptr{Ptr{p2est_quadrant_t}}, Cint, Cint, + Ptr{Ptr{p4est_quadrant_t}}, Ptr{Ptr{p2est_quadrant_t}})) + + refine_column_fn_callback_c, refine_column_replace_callback_c end + function p6est_vertically_coarsen_callbacks() function coarsen_layer_callback(p6est::Ptr{p6est_t}, which_tree::p4est_topidx_t, @@ -510,16 +623,13 @@ function p6est_vertically_coarsen_callbacks() # of the refininement process have quadrant.p.user_data == C_NULL # Not sure why ... The following if-end takes care of this. if (quadrant.p.user_data) == C_NULL - println("XXX: first if") return Cint(0) end is_coarsen_flag=(unsafe_wrap(Array,Ptr{Cint}(quadrant.p.user_data),1)[])==coarsen_flag if (!is_coarsen_flag) - println("XXX: second if") return Cint(0) end end - println("XXX: function end") return coarsen end coarsen_fn_callback_c = @cfunction($coarsen_layer_callback, @@ -529,6 +639,39 @@ function p6est_vertically_coarsen_callbacks() return coarsen_fn_callback_c end + +function p6est_horizontally_coarsen_callbacks() + function coarsen_column_callback(p6est::Ptr{p6est_t}, + which_tree::p4est_topidx_t, + columns::Ptr{Ptr{p4est_quadrant_t}}) + + forest=p6est[] + num_children=4 + columns_array=unsafe_wrap(Array, columns, num_children) + + coarsen=Cint(1) + for quadrant_index=1:num_children + column = columns_array[quadrant_index][] + f,l = P6EST_COLUMN_GET_RANGE(column) + q2_ptr = p2est_quadrant_array_index(forest.layers[], f) + if (q2_ptr[].p.user_data) == C_NULL + return Cint(0) + end + is_coarsen_flag = (unsafe_wrap(Array,Ptr{Cint}(q2_ptr[].p.user_data),1)[])==coarsen_flag + if (!is_coarsen_flag) + return Cint(0) + end + end + return coarsen + end + coarsen_fn_callback_c = @cfunction($coarsen_column_callback, + Cint, + (Ptr{p6est_t}, p4est_topidx_t, Ptr{Ptr{p4est_quadrant_t}})) + + return coarsen_fn_callback_c +end + + function pXest_coarsen_callbacks(::P4estType) function coarsen_callback(forest_ptr::Ptr{p4est_t}, which_tree::p4est_topidx_t, @@ -715,6 +858,7 @@ function setup_cell_prange(pXest_type::PXestType, k=k+1 end end + @debug "[$(MPI.Comm_rank(MPI.COMM_WORLD))] gho_to_glo=$(gho_to_glo) gho_to_own=$(gho_to_own)" global_first_quadrant[part+1]-global_first_quadrant[part],global_first_quadrant[part]+1,gho_to_glo,gho_to_own end |> tuple_of_arrays ngids = global_first_quadrant[end]+1 @@ -828,7 +972,7 @@ function p6est_lnodes_decode(face_code, work = Int16(face_code >> 5) hanging_face .= -1 hanging_edge .= -1 - p4est_lnodes_decode(fc4, hanging_face[3:end]) + p4est_lnodes_decode(fc4, view(hanging_face,3:length(hanging_face))) for f=0:3 hf = hanging_face[f + 3] w = work & 0x0001 @@ -1295,6 +1439,8 @@ function generate_cell_faces_and_non_conforming_glue(pXest_type::PXestType, p4est_lvertex = p8est_edge_corners[p4est_ledge, hanging_vertex_lvertex_within_edge+1] gridap_cell_vertices[PXEST_2_GRIDAP_VERTEX[p4est_lvertex+1]] = hanging_vertex_code + + @debug "[$(MPI.Comm_rank(MPI.COMM_WORLD))] cell=$(cell) hanging p4est_ledge=$(p4est_ledge) half=$(half) hanging p4est_lvertex=$(PXEST_2_GRIDAP_VERTEX[p4est_lvertex+1])" end end end @@ -1357,7 +1503,7 @@ function generate_cell_faces_and_non_conforming_glue(pXest_type::PXestType, hanging_faces_pairs_to_owner_face[(cell, PXEST_2_GRIDAP_FACE[p4est_lface])] = (owner_face,half+1) else # Anisotropic refinement - @assert half in 4:7 + @assert half in 4:7 for regular_vertex_lvertex_within_face in p6est_half_to_regular_vertices[mod(half,4)+1,:] # Process regular vertex p4est_regular_lvertex = pXest_face_corners[p4est_lface, regular_vertex_lvertex_within_face+1] @@ -1377,8 +1523,6 @@ function generate_cell_faces_and_non_conforming_glue(pXest_type::PXestType, hanging_faces_pairs_to_owner_face[(cell, PXEST_2_GRIDAP_FACE[p4est_lface])] = (owner_face,subface) end - - if (Dc==3) _subface_to_hanging_edges_within_subface = subface_to_hanging_edges_within_subface(pXest_type) _subface_to_hanging_edges_within_face = subface_to_hanging_edges_within_face(pXest_type) @@ -1386,7 +1530,7 @@ function generate_cell_faces_and_non_conforming_glue(pXest_type::PXestType, for (i,ledge_within_face) in enumerate(_subface_to_hanging_edges_within_subface[mod(half,4)+1,:]) p4est_ledge=p8est_face_edges[p4est_lface,ledge_within_face+1] - @debug "[$(MPI.Comm_rank(MPI.COMM_WORLD))] cell=$(cell) p4est_lface=$(p4est_lface) half=$(half) index=$(mod(half,4)+1) p4est_ledge=$(p4est_ledge) ledge_within_face=$(ledge_within_face)" + @debug "[$(MPI.Comm_rank(MPI.COMM_WORLD))] cell=$(cell) p4est_lface=$(p4est_lface) half=$(half) index=$(mod(half,4)+1) p4est_ledge=$(p4est_ledge) ledge_within_face=$(ledge_within_face) " gridap_ledge = PXEST_2_GRIDAP_EDGE[p4est_ledge+1] # Identify the two edges which are hanging within the face @@ -1463,6 +1607,9 @@ function generate_cell_faces_and_non_conforming_glue(pXest_type::PXestType, # Process hanging edge subedge = regular_vertex_lvertex_within_edge+1 owner_edge_subedge_pair=(p4est_owner_edge,subedge) + + @debug "[$(MPI.Comm_rank(MPI.COMM_WORLD))] cell=$(cell) hanging_edge=$(hanging_edge) p4est_ledge=$(p4est_ledge) gridap_cell_edges[PXEST_2_GRIDAP_EDGE[p4est_ledge]]: $(gridap_cell_edges[PXEST_2_GRIDAP_EDGE[p4est_ledge]]) half=$(half) owner_edge_subedge_pair=$(owner_edge_subedge_pair)" + gridap_ledge=PXEST_2_GRIDAP_EDGE[p4est_ledge] hanging_edges_cell_ledge_to_owner_face_half[(cell, gridap_ledge)] = owner_edge_subedge_pair if (!haskey(owner_edge_subedge_to_cell_ledge,owner_edge_subedge_pair)) @@ -1569,49 +1716,62 @@ function generate_cell_faces_and_non_conforming_glue(pXest_type::PXestType, owner_p4est_gedge_subedge_to_hanging_edge = Dict{Tuple{Int,Int},Int}() num_hanging_faces[2] = 0 ledge_to_cvertices = Gridap.ReferenceFEs.get_faces(HEX, 1, 0) - # The following loop needs (1) the pairs to be traversed in increased order by cell ID; - # (2) gridap cell vertices to be already completed - for key in sort(collect(keys(hanging_edges_cell_ledge_to_owner_face_half))) + # The following loop needs gridap cell vertices to be already completed + for key in keys(hanging_edges_cell_ledge_to_owner_face_half) + (cell, ledge) = key + (owner_p4est_gface_or_gedge, half) = hanging_edges_cell_ledge_to_owner_face_half[key] + if (half<0) # hanging edge is within a coarser face + owner_p4est_gface = owner_p4est_gface_or_gedge + if !(haskey(owner_p4est_gface_half_to_hanging_edge, (owner_p4est_gface,half))) + num_hanging_faces[2] += 1 + owner_p4est_gface_half_to_hanging_edge[(owner_p4est_gface,half)] = num_hanging_faces[2] + if (!is_ghost(cell) || (haskey(regular_faces_p4est_to_gridap,owner_p4est_gface))) + (owner_cell, p4est_lface) = p4est_gface_to_gcell_p4est_lface[owner_p4est_gface] + push!(hanging_edges_owner_cell_and_lface, + (owner_cell, n_cell_vertices+n_cell_edges+PXEST_2_GRIDAP_FACE[p4est_lface],half)) + else + push!(hanging_edges_owner_cell_and_lface,(-1, -1, -1)) + end + end + start_gridap_edges = (cell - 1) * n_cell_edges + gridap_cell_edges_data[start_gridap_edges+ledge] = num_regular_faces[2] + + owner_p4est_gface_half_to_hanging_edge[(owner_p4est_gface,half)] + else # hanging edge is within a coarser edge + @assert half==1 || half==2 + owner_p4est_gedge = owner_p4est_gface_or_gedge + owner_gedge_pair = (owner_p4est_gedge,half) + if (haskey(owner_edge_subedge_to_cell_ledge,owner_gedge_pair)) + (owner_cell, owner_cell_ledge) = owner_edge_subedge_to_cell_ledge[owner_gedge_pair] + if (owner_cell==cell) + @assert owner_cell_ledge == ledge + num_hanging_faces[2] += 1 + owner_p4est_gedge_subedge_to_hanging_edge[(owner_p4est_gedge,half)] = num_hanging_faces[2] + if (!is_ghost(cell) || (haskey(regular_edges_p4est_to_gridap,owner_p4est_gedge))) + (owner_cell, p4est_ledge) = p4est_gedge_to_gcell_p4est_ledge[owner_p4est_gedge] + push!(hanging_edges_owner_cell_and_lface, + (owner_cell, n_cell_vertices+PXEST_2_GRIDAP_EDGE[p4est_ledge],half)) + else + push!(hanging_edges_owner_cell_and_lface,(-1, -1, -1)) + end + start_gridap_edges = (cell - 1) * n_cell_edges + gridap_cell_edges_data[start_gridap_edges+ledge] = num_regular_faces[2] + num_hanging_faces[2] + end + end + end + end + @debug "[$(MPI.Comm_rank(MPI.COMM_WORLD))] gridap_cell_edges_data: $(gridap_cell_edges_data)" + + for key in keys(hanging_edges_cell_ledge_to_owner_face_half) (cell, ledge) = key (owner_p4est_gface_or_gedge, half) = hanging_edges_cell_ledge_to_owner_face_half[key] @debug "[$(MPI.Comm_rank(MPI.COMM_WORLD))] own_length=$(own_length(indices)) cell=$(cell) ledge=$(ledge) owner_p4est_gface_or_gedge=$(owner_p4est_gface_or_gedge) half=$(half)" - if (half<0) # hanging edge is within a coarser face - owner_p4est_gface = owner_p4est_gface_or_gedge - if !(haskey(owner_p4est_gface_half_to_hanging_edge, (owner_p4est_gface,half))) - num_hanging_faces[2] += 1 - owner_p4est_gface_half_to_hanging_edge[(owner_p4est_gface,half)] = num_hanging_faces[2] - if (!is_ghost(cell) || (haskey(regular_faces_p4est_to_gridap,owner_p4est_gface))) - (owner_cell, p4est_lface) = p4est_gface_to_gcell_p4est_lface[owner_p4est_gface] - push!(hanging_edges_owner_cell_and_lface, - (owner_cell, n_cell_vertices+n_cell_edges+PXEST_2_GRIDAP_FACE[p4est_lface],half)) - else - push!(hanging_edges_owner_cell_and_lface,(-1, -1, -1)) - end - end - start_gridap_edges = (cell - 1) * n_cell_edges - gridap_cell_edges_data[start_gridap_edges+ledge] = num_regular_faces[2] + - owner_p4est_gface_half_to_hanging_edge[(owner_p4est_gface,half)] - - else # hanging edge is within a coarser edge + if (half>0) # hanging edge is within a coarser face @assert half==1 || half==2 owner_p4est_gedge = owner_p4est_gface_or_gedge owner_gedge_pair = (owner_p4est_gedge,half) if (haskey(owner_edge_subedge_to_cell_ledge,owner_gedge_pair)) (owner_cell, owner_cell_ledge) = owner_edge_subedge_to_cell_ledge[owner_gedge_pair] - if (owner_cell==cell) - @assert owner_cell_ledge == ledge - num_hanging_faces[2] += 1 - owner_p4est_gedge_subedge_to_hanging_edge[(owner_p4est_gedge,half)] = num_hanging_faces[2] - if (!is_ghost(cell) || (haskey(regular_edges_p4est_to_gridap,owner_p4est_gedge))) - (owner_cell, p4est_ledge) = p4est_gedge_to_gcell_p4est_ledge[owner_p4est_gedge] - push!(hanging_edges_owner_cell_and_lface, - (owner_cell, n_cell_vertices+PXEST_2_GRIDAP_EDGE[p4est_ledge],half)) - else - push!(hanging_edges_owner_cell_and_lface,(-1, -1, -1)) - end - start_gridap_edges = (cell - 1) * n_cell_edges - gridap_cell_edges_data[start_gridap_edges+ledge] = num_regular_faces[2] + num_hanging_faces[2] - else + if (owner_cell!=cell) haskey_first_subedge = haskey(owner_p4est_gedge_subedge_to_hanging_edge,(owner_p4est_gedge,1)) haskey_second_subedge = haskey(owner_p4est_gedge_subedge_to_hanging_edge,(owner_p4est_gedge,2)) if (!(is_ghost(cell))) @@ -1650,19 +1810,18 @@ function generate_cell_faces_and_non_conforming_glue(pXest_type::PXestType, owner_half=1 elseif (haskey_second_subedge) owner_half=2 - end + end @debug "[$(MPI.Comm_rank(MPI.COMM_WORLD))] cell=$(cell) ledge=$(ledge) owner_p4est_gface_or_gedge=$(owner_p4est_gface_or_gedge) half=$(half) owner_half=$(owner_half)" start_gridap_edges = (cell - 1) * n_cell_edges gridap_cell_edges_data[start_gridap_edges+ledge] = num_regular_faces[2] + owner_p4est_gedge_subedge_to_hanging_edge[(owner_p4est_gedge,owner_half)] end end - end - end + end + end @debug "[$(MPI.Comm_rank(MPI.COMM_WORLD))] gridap_cell_edges_data: $(gridap_cell_edges_data)" end - gridap_cell_faces = Vector{JaggedArray}(undef,Dc) gridap_cell_faces[1] = JaggedArray(gridap_cell_vertices_data,gridap_cell_vertices_ptrs) if (Dc==3) @@ -1849,12 +2008,105 @@ function pXest_get_quadrant_vertex_coordinates(::P8estType, pvxy) end + + function _fill_ghost_cells_node_coordinates!(pXest_type::P6estType, + PXEST_CORNERS, + vxy, + pvxy, + node_coordinates, + current, + cell_lids, + ptr_pXest_connectivity, + pXest_ghost) + + + function sc_array_p4est_locidx_t_index(sc_array_object::sc_array_t, it) + @assert sc_array_object.elem_size == sizeof(p4est_locidx_t) + @assert it in 0:sc_array_object.elem_count + ptr=Ptr{p4est_locidx_t}(sc_array_object.array + sc_array_object.elem_size*it) + return unsafe_wrap(Array, ptr, 1)[] + end + + Dc = num_cell_dims(pXest_type) + + column_ghost = pXest_ghost.column_ghost[] + ptr_p2est_ghost_quadrants = _unwrap_ghost_quadrants(pXest_type, pXest_ghost) + ptr_p4est_ghost_quadrants = _unwrap_ghost_quadrants(P4estType(), column_ghost) + + tree_offsets = unsafe_wrap(Array, column_ghost.tree_offsets, pXest_ghost.num_trees+1) + + current_ghost_column=0 + + # Go over ghost cells + for i=1:pXest_ghost.num_trees + for j=tree_offsets[i]:tree_offsets[i+1]-1 + p4est_quadrant = ptr_p4est_ghost_quadrants[j+1] + k = sc_array_p4est_locidx_t_index(pXest_ghost.column_layer_offsets[],current_ghost_column) + l = sc_array_p4est_locidx_t_index(pXest_ghost.column_layer_offsets[],current_ghost_column+1) + for m=k:l-1 + p2est_quadrant = ptr_p2est_ghost_quadrants[m+1] + coords=pXest_cell_coords(pXest_type,p4est_quadrant,p2est_quadrant) + levels=pXest_get_quadrant_and_layer_levels(pXest_type,p4est_quadrant,p2est_quadrant) + for vertex=1:PXEST_CORNERS + pXest_get_quadrant_vertex_coordinates(pXest_type, + ptr_pXest_connectivity, + p4est_topidx_t(i-1), + coords, + levels, + Cint(vertex-1), + pvxy) + node_coordinates[cell_lids[current]]=Point{Dc,Float64}(vxy...) + current=current+1 + end + end + current_ghost_column=current_ghost_column+1 + end + end + end + +function _fill_ghost_cells_node_coordinates!(pXest_type::P4P8estType, + PXEST_CORNERS, + vxy, + pvxy, + node_coordinates, + current, + cell_lids, + ptr_pXest_connectivity, + pXest_ghost) + + Dc = num_cell_dims(pXest_type) + + tree_offsets = unsafe_wrap(Array, pXest_ghost.tree_offsets, pXest_ghost.num_trees+1) + ptr_ghost_quadrants = _unwrap_ghost_quadrants(pXest_type, pXest_ghost) + + # Go over ghost cells + for i=1:pXest_ghost.num_trees + for j=tree_offsets[i]:tree_offsets[i+1]-1 + quadrant = ptr_ghost_quadrants[j+1] + coords=pXest_cell_coords(pXest_type,quadrant,0) + levels=pXest_get_quadrant_and_layer_levels(pXest_type,quadrant,0) + for vertex=1:PXEST_CORNERS + pXest_get_quadrant_vertex_coordinates(pXest_type, + ptr_pXest_connectivity, + p4est_topidx_t(i-1), + coords, + levels, + Cint(vertex-1), + pvxy) + node_coordinates[cell_lids[current]]=Point{Dc,Float64}(vxy...) + current=current+1 + end + end + end +end + + function generate_node_coordinates(pXest_type::PXestType, - cell_vertex_lids, - nlvertices, - ptr_pXest_connectivity, - ptr_pXest, - ptr_pXest_ghost) + cell_vertex_lids, + nlvertices, + ptr_pXest_connectivity, + ptr_pXest, + ptr_pXest_ghost) Dc = num_cell_dims(pXest_type) @@ -1862,10 +2114,6 @@ function generate_node_coordinates(pXest_type::PXestType, pXest_ghost = ptr_pXest_ghost[] pXest = ptr_pXest[] - # Obtain ghost quadrants - ptr_ghost_quadrants = _unwrap_ghost_quadrants(pXest_type, pXest_ghost) - - tree_offsets = unsafe_wrap(Array, pXest_ghost.tree_offsets, pXest_ghost.num_trees+1) dnode_coordinates=map(cell_vertex_lids,nlvertices) do cell_vertex_lids, nl node_coordinates=Vector{Point{Dc,Float64}}(undef,nl) current=1 @@ -1893,45 +2141,24 @@ function generate_node_coordinates(pXest_type::PXestType, pvxy) - @debug "[$(MPI.Comm_rank(MPI.COMM_WORLD))] quadrant=$(cell) layer=$(l) coords=$(coords) levels=$(levels) pvxy=$(unsafe_wrap(Array, pvxy, 3))" + @debug "[$(MPI.Comm_rank(MPI.COMM_WORLD))] quadrant=$(cell) layer=$(l) coords=$(coords) levels=$(levels) pvxy=$(unsafe_wrap(Array, pvxy, 3)) cell_lids=$(cell_lids[current])" node_coordinates[cell_lids[current]]=Point{Dc,Float64}(vxy...) current=current+1 end - end end end + _fill_ghost_cells_node_coordinates!(pXest_type, + PXEST_CORNERS, + vxy, + pvxy, + node_coordinates, + current, + cell_lids, + ptr_pXest_connectivity, + pXest_ghost) - # Go over ghost cells - for i=1:pXest_ghost.num_trees - for j=tree_offsets[i]:tree_offsets[i+1]-1 - quadrant = ptr_ghost_quadrants[j+1] - for vertex=1:PXEST_CORNERS - if (Dc==2) - p4est_get_quadrant_vertex_coordinates(ptr_pXest_connectivity, - p4est_topidx_t(i-1), - quadrant.x, - quadrant.y, - quadrant.level, - Cint(vertex-1), - pvxy) - else - p8est_get_quadrant_vertex_coordinates(ptr_pXest_connectivity, - p4est_topidx_t(i-1), - quadrant.x, - quadrant.y, - quadrant.z, - quadrant.level, - Cint(vertex-1), - pvxy) - - end - node_coordinates[cell_lids[current]]=Point{Dc,Float64}(vxy...) - current=current+1 - end - end - end node_coordinates end end @@ -2044,6 +2271,8 @@ function pXest_compute_migration_control_data(pXest_type::P6estType,ptr_pXest_ol pXest_new.columns) ptrs=Vector{Int32}(undef,length(columns_snd_lids.ptrs)) + ptrs.=0 + lids=Int32[] old2new=Int32[] @@ -2053,6 +2282,7 @@ function pXest_compute_migration_control_data(pXest_type::P6estType,ptr_pXest_ol current_rank = 1 num_trees = Cint(pXest_old.columns[].connectivity[].num_trees) + @assert num_trees == Cint(pXest_new.columns[].connectivity[].num_trees) # Go over trees for itree = 0:num_trees-1 @@ -2064,18 +2294,46 @@ function pXest_compute_migration_control_data(pXest_type::P6estType,ptr_pXest_ol q = pXest_quadrant_array_index(pXest_type,tree,iquad) f,l=P6EST_COLUMN_GET_RANGE(q[]) num_quads_in_column=l-f - if (columns_old2new[current_col]==0) - if (current_col==columns_snd_lids.data[columns_snd_lids.ptrs[current_rank+1]]) - current_rank=current_rank+1 - end + current_new_col=columns_old2new[current_col] + if (current_new_col==0) + if ((current_rank+1)!=length(columns_snd_lids.ptrs)) + if (current_col==columns_snd_lids.data[columns_snd_lids.ptrs[current_rank+1]]) + current_rank=current_rank+1 + end + end ptrs[current_rank+1]+=num_quads_in_column for i=0:num_quads_in_column-1 push!(lids,current_old_cell+i) + push!(old2new,0) end else + # Count how many quads are in previous columns! + current_new_cell=1 + col=1 + found=false + # Go over trees + for jtree = 0:num_trees-1 + tree_new = pXest_tree_array_index(pXest_type,pXest_new,jtree)[] + num_quads_new = Cint(tree_new.quadrants.elem_count) + # Go over columns of current tree + for jquad=0:num_quads_new-1 + if (col==current_new_col) + found=true + break + end + q = pXest_quadrant_array_index(pXest_type,tree_new,jquad) + f,l=P6EST_COLUMN_GET_RANGE(q[]) + col+=1 + current_new_cell+=l-f + end + if (found) + break + end + end + @assert found for i=1:num_quads_in_column - push!(old2new,current_new_cell) - current_new_cell+=1 + push!(old2new,current_new_cell) + current_new_cell+=1 end end current_old_cell+=num_quads_in_column @@ -2134,4 +2392,29 @@ function pXest_inflate(::P8estType, quadrants, data, user_pointer) +end + +function pXest_stride_among_children(::P4P8estType, + ::PXestUniformRefinementRuleType, + ptr_pXest) + return 1 +end + +function pXest_stride_among_children(::P6estType, + ::PXestVerticalRefinementRuleType, + ptr_pXest) + return 1 +end + +function pXest_stride_among_children(pXest_type::P6estType, + ::PXestHorizontalRefinementRuleType, + ptr_pXest) + # Here we are assuming: + # (1) Each processor has at least one column. + # (2) The number of layers in each column is the same within and accross processors. + pXest = ptr_pXest[] + tree = pXest_tree_array_index(pXest_type, pXest, 0)[] + q = pXest_quadrant_array_index(pXest_type, tree, 0) + f,l=P6EST_COLUMN_GET_RANGE(q[]) + return l-f end \ No newline at end of file diff --git a/src/UniformlyRefinedForestOfOctreesDiscreteModels.jl b/src/UniformlyRefinedForestOfOctreesDiscreteModels.jl index b33cc70..1c825b3 100644 --- a/src/UniformlyRefinedForestOfOctreesDiscreteModels.jl +++ b/src/UniformlyRefinedForestOfOctreesDiscreteModels.jl @@ -576,11 +576,6 @@ function generate_face_labeling(pXest_type::P4P8estType, facet_to_entity = map(x->x[Dc] , faces_to_entity) cell_to_entity = map(x->x[Dc+1], faces_to_entity) - function cell_to_faces(topology,cell_dim,face_dim) - map(topology) do topology - Gridap.Geometry.get_faces(topology,cell_dim,face_dim) - end - end polytope = Dc==2 ? QUAD : HEX @@ -637,6 +632,12 @@ function generate_face_labeling(pXest_type::P4P8estType, face_labeling end +function cell_to_faces(topology,cell_dim,face_dim) + map(topology) do topology + Gridap.Geometry.get_faces(topology,cell_dim,face_dim) + end +end + function _fill_data!(data,entry::Integer,k) data[k]=entry k=k+1 diff --git a/test/PoissonAnisotropicOctreeModelsTests.jl b/test/PoissonAnisotropicOctreeModelsTests.jl index b952b7b..c480559 100644 --- a/test/PoissonAnisotropicOctreeModelsTests.jl +++ b/test/PoissonAnisotropicOctreeModelsTests.jl @@ -33,20 +33,23 @@ module PoissonAnisotropicOctreeModelsTests reffe=ReferenceFE(lagrangian,T,order) VH=FESpace(dmodel,reffe;dirichlet_tags="boundary") UH=TrialFESpace(VH,u) - ref_coarse_flags=map(ranks,partition(get_cell_gids(dmodel.dmodel))) do rank,indices - flags=zeros(Cint,length(indices)) + num_local_cols=GridapP4est.num_locally_owned_columns(dmodel) + ref_coarse_flags=map(ranks,num_local_cols) do rank,num_local_cols + flags=zeros(Cint,num_local_cols) flags.=nothing_flag flags[1]=refine_flag - flags[own_length(indices)]=refine_flag + flags[end]=refine_flag # To create some unbalance - if (rank%2==0 && own_length(indices)>1) - flags[div(own_length(indices),2)]=refine_flag + if (rank%2==0) + flags[div(num_local_cols,2)]=refine_flag end flags end - fmodel,glue=GridapP4est.vertically_adapt(dmodel,ref_coarse_flags); + fmodel,glue=GridapP4est.horizontally_adapt(dmodel,ref_coarse_flags); + + writevtk(fmodel,"fmodel") Vh=FESpace(fmodel,reffe,conformity=:H1;dirichlet_tags="boundary") Uh=TrialFESpace(Vh,u) @@ -80,16 +83,17 @@ module PoissonAnisotropicOctreeModelsTests op = AffineFEOperator(ah,bh,Uh,Vh) uh = solve(op) + uh2dofs=get_free_dof_values(interpolate(u, Uh)) + # println(op.op.matrix.matrix_partition.item_ref[]*uh2dofs.vector_partition.item_ref[]- - # op.op.vector.vector_partition.item_ref[]) - #writevtk(ΩH, "ctrian", cellfields=["uH"=>uH]) - #writevtk(Ωh, "ftrian", cellfields=["uh"=>uh]) + # op.op.vector.vector_partition.item_ref[]) + # writevtk(ΩH, "ctrian", cellfields=["uH"=>uH]) + # writevtk(Ωh, "ftrian", cellfields=["uh"=>uh]) # # Compute errors e = u - uh el2 = sqrt(sum( ∫( e⋅e )*dΩh )) - eh1 = sqrt(sum( ∫( e⋅e + ∇(e)⊙∇(e) )*dΩh )) - + eh1 = sqrt(sum( ∫( e⋅e + ∇(e)⊙∇(e) )*dΩh )) println("[SOLVE FINE] el2 < tol: $(el2) < $(tol)") println("[SOLVE FINE] eh1 < tol: $(eh1) < $(tol)") @assert el2 < tol @@ -146,12 +150,12 @@ module PoissonAnisotropicOctreeModelsTests println("[SOLVE FINE REDISTRIBUTED] el2 < tol: $(el2) < $(tol)") @assert el2 < tol + uhred2 = GridapDistributed.redistribute_fe_function(uh,Vhred,fmodel_red,red_glue) + e = u - uhred2 - uhred2 = GridapDistributed.redistribute_fe_function(uh,Vhred,fmodel_red,red_glue) - e = u - uhred2 - el2 = sqrt(sum( ∫( e⋅e )*dΩhred )) - println("[REDISTRIBUTE SOLUTION] el2 < tol: $(el2) < $(tol)") - @assert el2 < tol + el2 = sqrt(sum( ∫( e⋅e )*dΩhred )) + println("[REDISTRIBUTE SOLUTION] el2 < tol: $(el2) < $(tol)") + @assert el2 < tol fmodel_red end @@ -164,18 +168,20 @@ module PoissonAnisotropicOctreeModelsTests # Define manufactured functions u,f = generate_analytical_problem_functions(T,order) + num_local_cols=GridapP4est.num_locally_owned_columns(dmodel) + degree = 2*order+1 - ref_coarse_flags=map(ranks,partition(get_cell_gids(dmodel.dmodel))) do rank,indices - flags=zeros(Int,length(indices)) + ref_coarse_flags=map(ranks,num_local_cols) do rank,num_local_cols + flags=zeros(Int,num_local_cols) flags.=nothing_flag - if (rank==1) - flags[1:2].=coarsen_flag - else - flags[own_length(indices)]=refine_flag - end + #if (rank==1) + # flags[1:4].=coarsen_flag + #else + flags[1]=refine_flag + #end flags end - fmodel,glue=GridapP4est.vertically_adapt(dmodel,ref_coarse_flags); + fmodel,glue=GridapP4est.horizontally_adapt(dmodel,ref_coarse_flags); reffe=ReferenceFE(lagrangian,T,order) VH=FESpace(dmodel,reffe,conformity=:H1;dirichlet_tags="boundary") @@ -232,7 +238,8 @@ module PoissonAnisotropicOctreeModelsTests function test_3d(ranks,order,T::Type;num_amr_steps=5) coarse_model=CartesianDiscreteModel((0,1,0,1),(1,1)) dmodel=AnisotropicallyAdapted3DDistributedDiscreteModel(ranks, coarse_model, 1, 1) - test_refine_and_coarsen_at_once(ranks,dmodel,order,T) + #writevtk(dmodel.dmodel,"model") + #test_refine_and_coarsen_at_once(ranks,dmodel,order,T) rdmodel=dmodel for i=1:num_amr_steps rdmodel=test_transfer_ops_and_redistribute(ranks,rdmodel,order,T) @@ -254,8 +261,8 @@ module PoissonAnisotropicOctreeModelsTests end end function run(distribute) - #debug_logger = ConsoleLogger(stderr, Logging.Debug) - #global_logger(debug_logger); # Enable the debug logger globally + debug_logger = ConsoleLogger(stderr, Logging.Debug) + global_logger(debug_logger); # Enable the debug logger globally ranks = distribute(LinearIndices((MPI.Comm_size(MPI.COMM_WORLD),))) for perm=1:4, order=1:4, scalar_or_vector in (:scalar,) test(ranks,perm,order,_field_type(Val{3}(),scalar_or_vector)) @@ -263,7 +270,10 @@ module PoissonAnisotropicOctreeModelsTests for perm in (1,2), order in (1,4), scalar_or_vector in (:vector,) test(ranks,perm,order,_field_type(Val{3}(),scalar_or_vector)) end - for order=2:2, scalar_or_vector in (:scalar,:vector) + for order=1:1, scalar_or_vector in (:scalar,:vector) + test_3d(ranks,order,_field_type(Val{3}(),scalar_or_vector), num_amr_steps=4) + end + for order=1:1, scalar_or_vector in (:scalar,) test_3d(ranks,order,_field_type(Val{3}(),scalar_or_vector), num_amr_steps=4) end end diff --git a/test/PoissonNonConformingOctreeModelsTests.jl b/test/PoissonNonConformingOctreeModelsTests.jl index 3c9fdfa..7ec3cb8 100644 --- a/test/PoissonNonConformingOctreeModelsTests.jl +++ b/test/PoissonNonConformingOctreeModelsTests.jl @@ -52,7 +52,7 @@ module PoissonNonConformingOctreeModelsTests end flags end - fmodel,glue=adapt(dmodel,ref_coarse_flags); + fmodel,glue=Gridap.Adaptivity.adapt(dmodel,ref_coarse_flags); # map(ranks,glue) do rank, glue # if rank==2 # print(glue.n2o_faces_map[end]); print("\n") @@ -185,7 +185,7 @@ module PoissonNonConformingOctreeModelsTests end flags end - fmodel,glue=adapt(dmodel,ref_coarse_flags); + fmodel,glue=Gridap.Adaptivity.adapt(dmodel,ref_coarse_flags); reffe=ReferenceFE(lagrangian,T,order) VH=FESpace(dmodel,reffe,conformity=:H1;dirichlet_tags="boundary") From e62494d5d6cb6b10d16db0c4893bc12aac488580 Mon Sep 17 00:00:00 2001 From: "Alberto F. Martin" Date: Sun, 3 Mar 2024 21:38:48 +1100 Subject: [PATCH 04/29] Adding Gridap.Adaptivity. to adapt --- test/AdaptivityFlagsMarkingStrategiesTests.jl | 2 +- test/DarcyNonConformingOctreeModelsTests.jl | 6 +++--- 2 files changed, 4 insertions(+), 4 deletions(-) diff --git a/test/AdaptivityFlagsMarkingStrategiesTests.jl b/test/AdaptivityFlagsMarkingStrategiesTests.jl index f317603..a6ffbf9 100644 --- a/test/AdaptivityFlagsMarkingStrategiesTests.jl +++ b/test/AdaptivityFlagsMarkingStrategiesTests.jl @@ -53,7 +53,7 @@ module AdaptivityFlagsMarkingStrategiesTests error_indicators; verbose=true) - model,glue=adapt(dmodel,ref_coarse_flags); + model,glue=Gridap.Adaptivity.adapt(dmodel,ref_coarse_flags); # Define manufactured functions u(x) = x[1]+x[2]^order diff --git a/test/DarcyNonConformingOctreeModelsTests.jl b/test/DarcyNonConformingOctreeModelsTests.jl index bfe0c94..2d513cf 100644 --- a/test/DarcyNonConformingOctreeModelsTests.jl +++ b/test/DarcyNonConformingOctreeModelsTests.jl @@ -26,7 +26,7 @@ module DarcyNonConformingOctreeModelsTests end flags end - fmodel,glue=adapt(dmodel,ref_coarse_flags); + fmodel,glue=Gridap.Adaptivity.adapt(dmodel,ref_coarse_flags); # Solve coarse xH,XH=solve_darcy(dmodel,order) @@ -120,7 +120,7 @@ module DarcyNonConformingOctreeModelsTests end flags end - fmodel,glue=adapt(dmodel,ref_coarse_flags); + fmodel,glue=Gridap.Adaptivity.adapt(dmodel,ref_coarse_flags); # Solve coarse xH,XH=solve_darcy(dmodel,order) @@ -262,7 +262,7 @@ module DarcyNonConformingOctreeModelsTests flags[own_length(indices)]=refine_flag flags end - fmodel,glue=adapt(model,ref_coarse_flags) + fmodel,glue=Gridap.Adaptivity.adapt(model,ref_coarse_flags) xh,Xh = solve_darcy(fmodel,order) check_error_darcy(fmodel,order,xh) end From b6321f167db3e840d2d7d56c7b8678d0777e0855 Mon Sep 17 00:00:00 2001 From: "Alberto F. Martin" Date: Tue, 5 Mar 2024 22:21:50 +1100 Subject: [PATCH 05/29] * Some bug-fixes for 2D(adaptive)+1D(uniform) refinement * Added missing methods for uniformly refining/coarsening in the regular quad/octree --- ...callyAdapted3DDistributedDiscreteModels.jl | 66 +++++++++++-------- src/PXestTypeMethods.jl | 56 +++++++++++++--- test/PoissonAnisotropicOctreeModelsTests.jl | 8 ++- test/runtests.jl | 3 + 4 files changed, 96 insertions(+), 37 deletions(-) diff --git a/src/AnisotropicallyAdapted3DDistributedDiscreteModels.jl b/src/AnisotropicallyAdapted3DDistributedDiscreteModels.jl index 35a9e21..363a0b5 100644 --- a/src/AnisotropicallyAdapted3DDistributedDiscreteModels.jl +++ b/src/AnisotropicallyAdapted3DDistributedDiscreteModels.jl @@ -37,7 +37,7 @@ function AnisotropicallyAdapted3DDistributedDiscreteModel( ptr_pXest_connectivity, ptr_pXest, pXest_type, - nothing, + PXestHorizontalRefinementRuleType(), true, nothing) end @@ -407,11 +407,14 @@ function generate_face_labeling(pXest_type::P6estType, polytope = HEX + + update_face_to_entity_with_ghost_data!(vertex_to_entity, cell_prange, num_faces(polytope,0), cell_to_faces(topology,Dc,0)) + update_face_to_entity_with_ghost_data!(edget_to_entity, cell_prange, num_faces(polytope,1), @@ -421,11 +424,13 @@ function generate_face_labeling(pXest_type::P6estType, cell_prange, num_faces(polytope,Dc-1), cell_to_faces(topology,Dc,Dc-1)) + update_face_to_entity_with_ghost_data!(cell_to_entity, cell_prange, num_faces(polytope,Dc), cell_to_faces(topology,Dc,Dc)) + faces_to_entity=[vertex_to_entity,edget_to_entity,facet_to_entity,cell_to_entity] @@ -476,6 +481,21 @@ face_labeling = face_labeling end + + # map(partition(cell_prange)) do indices + # println("[$(MPI.Comm_rank(MPI.COMM_WORLD))] l2g=$(local_to_global(indices))") + # println("[$(MPI.Comm_rank(MPI.COMM_WORLD))] l2o=$(local_to_own(indices))") + # println("[$(MPI.Comm_rank(MPI.COMM_WORLD))] l2p=$(local_to_owner(indices))") + + + # cache=PartitionedArrays.assembly_cache(indices) + + # println("[$(MPI.Comm_rank(MPI.COMM_WORLD))] ns=$(cache.neighbors_snd[])") + # println("[$(MPI.Comm_rank(MPI.COMM_WORLD))] nr=$(cache.neighbors_rcv[])") + # println("[$(MPI.Comm_rank(MPI.COMM_WORLD))] li_snd=$(cache.local_indices_snd[])") + # println("[$(MPI.Comm_rank(MPI.COMM_WORLD))] li_rcv=$(cache.local_indices_rcv[])") + # end + face_labeling end @@ -507,9 +527,7 @@ function _horizontally_refine_coarsen_balance!(model::OctreeDistributedDiscreteM map(refinement_and_coarsening_flags,num_cols) do flags, num_cols # The length of the local flags array has to match the number of locally owned columns in the model @assert num_cols==length(flags) - println("BBB: $(flags)") pXest_reset_data!(pXest_type, model.ptr_pXest, Cint(sizeof(Cint)), init_fn_callback_c, pointer(flags)) - println("CCC: $(flags)") end @@ -559,16 +577,23 @@ function horizontally_adapt(model::OctreeDistributedDiscreteModel{Dc,Dp}, pXest_lnodes_destroy(model.pXest_type,ptr_pXest_lnodes) pXest_refinement_rule_type = PXestHorizontalRefinementRuleType() - _extruded_flags = _extrude_refinement_and_coarsening_flags(_refinement_and_coarsening_flags, - model.ptr_pXest, - ptr_new_pXest) + extruded_ref_coarsen_flags=map(partition(get_cell_gids(model)),refinement_and_coarsening_flags) do indices, flags + similar(flags, length(local_to_global(indices))) + end + + _extrude_refinement_and_coarsening_flags!(extruded_ref_coarsen_flags, + refinement_and_coarsening_flags, + model.ptr_pXest, + ptr_new_pXest) + stride = pXest_stride_among_children(model.pXest_type,pXest_refinement_rule_type,model.ptr_pXest) + adaptivity_glue = _compute_fine_to_coarse_model_glue(model.pXest_type, pXest_refinement_rule_type, model.parts, model.dmodel, fmodel, - _extruded_flags, + extruded_ref_coarsen_flags, stride) adaptive_models = map(local_views(model), local_views(fmodel), @@ -590,10 +615,11 @@ function horizontally_adapt(model::OctreeDistributedDiscreteModel{Dc,Dp}, return ref_model, adaptivity_glue end - function _extrude_refinement_and_coarsening_flags( - refinement_and_coarsening_flags::MPIArray{<:Vector{<:Integer}}, - ptr_pXest_old, - ptr_pXest_new) +function _extrude_refinement_and_coarsening_flags!( + extruded_flags::MPIArray{<:Vector{<:Integer}}, + flags::MPIArray{<:Vector{<:Integer}}, + ptr_pXest_old, + ptr_pXest_new) pXest_old = ptr_pXest_old[] pXest_new = ptr_pXest_new[] @@ -602,19 +628,9 @@ end num_trees = Cint(pXest_old.columns[].connectivity[].num_trees) @assert num_trees == Cint(pXest_new.columns[].connectivity[].num_trees) - map(refinement_and_coarsening_flags) do flags + map(flags,extruded_flags) do flags,extruded_flags current_old_quad=1 - current_cell_old=1 - total=0 - for itree=0:num_trees-1 - tree = pXest_tree_array_index(pXest_type,pXest_old,itree)[] - num_quads = Cint(tree.quadrants.elem_count) - q = pXest_quadrant_array_index(pXest_type,tree,0) - f,l=P6EST_COLUMN_GET_RANGE(q[]) - num_layers=l-f - total+=num_quads*num_layers - end - extruded_flags=similar(flags, total) + current_cell_old=1 # Go over trees for itree=0:num_trees-1 @@ -654,7 +670,5 @@ end end end end - - extruded_flags end -end \ No newline at end of file +end \ No newline at end of file diff --git a/src/PXestTypeMethods.jl b/src/PXestTypeMethods.jl index 97ba91c..7dd3205 100644 --- a/src/PXestTypeMethods.jl +++ b/src/PXestTypeMethods.jl @@ -2409,12 +2409,52 @@ end function pXest_stride_among_children(pXest_type::P6estType, ::PXestHorizontalRefinementRuleType, ptr_pXest) - # Here we are assuming: - # (1) Each processor has at least one column. - # (2) The number of layers in each column is the same within and accross processors. - pXest = ptr_pXest[] - tree = pXest_tree_array_index(pXest_type, pXest, 0)[] - q = pXest_quadrant_array_index(pXest_type, tree, 0) - f,l=P6EST_COLUMN_GET_RANGE(q[]) - return l-f + # Here we are assuming: + # (1) Each processor has at least one column. + # (2) The number of layers in each column is the same within and accross processors. + num_trees = ptr_pXest[].columns[].connectivity[].num_trees + for itree = 0:num_trees-1 + tree = pXest_tree_array_index(pXest_type, ptr_pXest[], itree)[] + if tree.quadrants.elem_count>0 + q = pXest_quadrant_array_index(pXest_type, tree, 0) + f,l=P6EST_COLUMN_GET_RANGE(q[]) + return l-f + end + end + return 0 +end + +function pXest_uniformly_refine!(::P4estType, ptr_pXest) + # Refine callbacks + function refine_fn_2d(::Ptr{p4est_t},which_tree::p4est_topidx_t,quadrant::Ptr{p4est_quadrant_t}) + return Cint(1) + end + refine_fn_c = @cfunction($refine_fn_2d,Cint,(Ptr{p4est_t}, p4est_topidx_t, Ptr{p4est_quadrant_t})) + p4est_refine(ptr_pXest, Cint(0), refine_fn_c, C_NULL) +end + +function pXest_uniformly_refine!(::P8estType, ptr_pXest) + # Refine callbacks + function refine_fn_3d(::Ptr{p8est_t},which_tree::p4est_topidx_t,quadrant::Ptr{p8est_quadrant_t}) + return Cint(1) + end + # C-callable refine callback 3D + refine_fn_c = @cfunction($refine_fn_3d,Cint,(Ptr{p8est_t}, p4est_topidx_t, Ptr{p8est_quadrant_t})) + p8est_refine(ptr_pXest, Cint(0), refine_fn_c, C_NULL) +end + +function pXest_uniformly_coarsen!(::P4estType, ptr_pXest) + function coarsen_fn_2d(::Ptr{p4est_t},::p4est_topidx_t,::Ptr{Ptr{p4est_quadrant_t}}) + return Cint(1) + end + coarsen_fn_c=@cfunction($coarsen_fn_2d,Cint,(Ptr{p4est_t}, p4est_topidx_t, Ptr{Ptr{p4est_quadrant_t}})) + p4est_coarsen(ptr_pXest, Cint(0), coarsen_fn_c, C_NULL) +end + +function pXest_uniformly_coarsen!(::P8estType, ptr_pXest) + function coarsen_fn_3d(::Ptr{p8est_t},::p4est_topidx_t,::Ptr{Ptr{p8est_quadrant_t}}) + return Cint(1) + end + coarsen_fn_c=@cfunction($coarsen_fn_3d,Cint,(Ptr{p8est_t}, p4est_topidx_t, Ptr{Ptr{p8est_quadrant_t}})) + p8est_coarsen(ptr_pXest, Cint(0), coarsen_fn_c, C_NULL) end \ No newline at end of file diff --git a/test/PoissonAnisotropicOctreeModelsTests.jl b/test/PoissonAnisotropicOctreeModelsTests.jl index c480559..16d1c79 100644 --- a/test/PoissonAnisotropicOctreeModelsTests.jl +++ b/test/PoissonAnisotropicOctreeModelsTests.jl @@ -35,6 +35,8 @@ module PoissonAnisotropicOctreeModelsTests UH=TrialFESpace(VH,u) num_local_cols=GridapP4est.num_locally_owned_columns(dmodel) ref_coarse_flags=map(ranks,num_local_cols) do rank,num_local_cols + println("[rank:$(rank)] $(num_local_cols)") + flags=zeros(Cint,num_local_cols) flags.=nothing_flag @@ -237,7 +239,7 @@ module PoissonAnisotropicOctreeModelsTests function test_3d(ranks,order,T::Type;num_amr_steps=5) coarse_model=CartesianDiscreteModel((0,1,0,1),(1,1)) - dmodel=AnisotropicallyAdapted3DDistributedDiscreteModel(ranks, coarse_model, 1, 1) + dmodel=AnisotropicallyAdapted3DDistributedDiscreteModel(ranks, coarse_model, 2, 2) #writevtk(dmodel.dmodel,"model") #test_refine_and_coarsen_at_once(ranks,dmodel,order,T) rdmodel=dmodel @@ -261,8 +263,8 @@ module PoissonAnisotropicOctreeModelsTests end end function run(distribute) - debug_logger = ConsoleLogger(stderr, Logging.Debug) - global_logger(debug_logger); # Enable the debug logger globally + # debug_logger = ConsoleLogger(stderr, Logging.Debug) + # global_logger(debug_logger); # Enable the debug logger globally ranks = distribute(LinearIndices((MPI.Comm_size(MPI.COMM_WORLD),))) for perm=1:4, order=1:4, scalar_or_vector in (:scalar,) test(ranks,perm,order,_field_type(Val{3}(),scalar_or_vector)) diff --git a/test/runtests.jl b/test/runtests.jl index 8a3f799..e8819a1 100644 --- a/test/runtests.jl +++ b/test/runtests.jl @@ -47,6 +47,9 @@ function run_tests(testdir) elseif f in ["PoissonNonConformingOctreeModelsTests.jl"] np = [1,2,4] extra_args = "" + elseif f in ["PoissonAnisotropicOctreeModelsTests.jl"] + np = [1,4] + extra_args = "" else np = [nprocs] extra_args = "" From 54c111ca25e5eaab220d51ce345ac6cee4d8b09f Mon Sep 17 00:00:00 2001 From: "Alberto F. Martin" Date: Wed, 6 Mar 2024 15:11:01 +1100 Subject: [PATCH 06/29] Additional bug-fixes --- src/OctreeDistributedDiscreteModels.jl | 5 ++++- src/PXestTypeMethods.jl | 13 ++++++++++--- test/OctreeDistributedDiscreteModelsTests.jl | 4 ++-- 3 files changed, 16 insertions(+), 6 deletions(-) diff --git a/src/OctreeDistributedDiscreteModels.jl b/src/OctreeDistributedDiscreteModels.jl index d49860c..3f950bf 100644 --- a/src/OctreeDistributedDiscreteModels.jl +++ b/src/OctreeDistributedDiscreteModels.jl @@ -1407,6 +1407,8 @@ function Gridap.Adaptivity.coarsen(model::OctreeDistributedDiscreteModel{Dc,Dp}) nc_glue=_create_conforming_model_non_conforming_glue(cmodel) + pXest_type = _dim_to_pXest_type(Dc) + c_octree_model = OctreeDistributedDiscreteModel(Dc,Dp, model.parts, cmodel, @@ -1414,6 +1416,7 @@ function Gridap.Adaptivity.coarsen(model::OctreeDistributedDiscreteModel{Dc,Dp}) model.coarse_model, model.ptr_pXest_connectivity, ptr_new_pXest, + pXest_type, PXestUniformRefinementRuleType(), false, model) @@ -1638,7 +1641,7 @@ function _redistribute_parts_supset_parts_redistributed( # "p4est_partition_cut_gloidx" function in the p4est library psub=PArrays.getany(parts_redistributed_model) Psub=GridapDistributed.num_parts(subset_comm) - first_global_quadrant=Int64((Float64(N)*Float64(psub-1))/(Float64(Psub))) + first_global_quadrant=Int64(floor((Float64(N)*Float64(psub-1))/(Float64(Psub)))) @assert first_global_quadrant>=0 && first_global_quadrant tuple_of_arrays - ngids = global_first_quadrant[end]+1 + ngids = global_first_quadrant[end] partition=map(parts,noids,firstgid,gho_to_glo,gho_to_own) do part, noids, firstgid, gho_to_glo, gho_to_own owner = part @@ -2457,4 +2456,12 @@ function pXest_uniformly_coarsen!(::P8estType, ptr_pXest) end coarsen_fn_c=@cfunction($coarsen_fn_3d,Cint,(Ptr{p8est_t}, p4est_topidx_t, Ptr{Ptr{p8est_quadrant_t}})) p8est_coarsen(ptr_pXest, Cint(0), coarsen_fn_c, C_NULL) -end \ No newline at end of file +end + +function pXest_partition_given!(::P4estType, ptr_pXest, new_num_cells_per_part) + p4est_partition_given(ptr_pXest, new_num_cells_per_part) +end + +function pXest_partition_given!(::P8estType, ptr_pXest, new_num_cells_per_part) + p8est_partition_given(ptr_pXest, new_num_cells_per_part) +end diff --git a/test/OctreeDistributedDiscreteModelsTests.jl b/test/OctreeDistributedDiscreteModelsTests.jl index c0eaaa3..c6446dd 100644 --- a/test/OctreeDistributedDiscreteModelsTests.jl +++ b/test/OctreeDistributedDiscreteModelsTests.jl @@ -87,7 +87,7 @@ module OctreeDistributedDiscreteModelsTests f_model_tasks_L2_back, dglueL1toL2 = redistribute(fmodel_tasks_L1,level_parts[2]) # Coarsening - model_back, glue = coarsen(f_model_tasks_L2_back) + model_back, glue = Gridap.Adaptivity.coarsen(f_model_tasks_L2_back) if GridapDistributed.i_am_in(level_parts[2]) @test num_cells(model_back)==num_cells(model) @@ -103,7 +103,7 @@ module OctreeDistributedDiscreteModelsTests model = OctreeDistributedDiscreteModel(level_parts[1],coarse_model,3) imodel = model for i=1:3 - omodel, glue = coarsen(imodel) + omodel, glue = Gridap.Adaptivity.coarsen(imodel) imodel = omodel end @test num_cells(imodel) == prod(nc) From a5fa023b4459b36e2ba90dd2244fe1cc060a8f18 Mon Sep 17 00:00:00 2001 From: "Alberto F. Martin" Date: Wed, 6 Mar 2024 17:46:27 +1100 Subject: [PATCH 07/29] Another bug-fix --- src/UniformlyRefinedForestOfOctreesDiscreteModels.jl | 3 +++ 1 file changed, 3 insertions(+) diff --git a/src/UniformlyRefinedForestOfOctreesDiscreteModels.jl b/src/UniformlyRefinedForestOfOctreesDiscreteModels.jl index 1c825b3..6776a62 100644 --- a/src/UniformlyRefinedForestOfOctreesDiscreteModels.jl +++ b/src/UniformlyRefinedForestOfOctreesDiscreteModels.jl @@ -759,6 +759,9 @@ function UniformlyRefinedForestOfOctreesDiscreteModel( num_uniform_refinements::Int) where {Dc,Dp} comm = parts.comm + + pXest_type = _dim_to_pXest_type(Dc) + ptr_pXest_connectivity, ptr_pXest, ptr_pXest_ghost, From c57a21c1edd85a343a5a14c6b9e81787bf8cb2fc Mon Sep 17 00:00:00 2001 From: "Alberto F. Martin" Date: Wed, 6 Mar 2024 19:58:43 +1100 Subject: [PATCH 08/29] Adding uniform refinement in the vertical direction --- ...callyAdapted3DDistributedDiscreteModels.jl | 75 +++++++++++++++++++ src/GridapP4est.jl | 1 + src/OctreeDistributedDiscreteModels.jl | 2 +- test/PoissonAnisotropicOctreeModelsTests.jl | 8 +- 4 files changed, 81 insertions(+), 5 deletions(-) diff --git a/src/AnisotropicallyAdapted3DDistributedDiscreteModels.jl b/src/AnisotropicallyAdapted3DDistributedDiscreteModels.jl index 363a0b5..10177bc 100644 --- a/src/AnisotropicallyAdapted3DDistributedDiscreteModels.jl +++ b/src/AnisotropicallyAdapted3DDistributedDiscreteModels.jl @@ -125,6 +125,81 @@ function vertically_adapt(model::OctreeDistributedDiscreteModel{Dc,Dp}, return ref_model, adaptivity_glue end +function _vertically_uniformly_refine!(model::OctreeDistributedDiscreteModel{Dc,Dp}) where {Dc,Dp} + pXest_type = model.pXest_type + function refine_layer_callback(p6est::Ptr{p6est_t}, + which_tree::p4est_topidx_t, + column::Ptr{p4est_quadrant_t}, + layer::Ptr{p2est_quadrant_t}) + Cint(1) + end + + refine_layer_fn_callback_c = @cfunction($refine_layer_callback, Cint, + (Ptr{p6est_t}, p4est_topidx_t, Ptr{p4est_quadrant_t}, Ptr{p2est_quadrant_t})) + + # # Copy input p4est, refine and balance + ptr_new_pXest = pXest_copy(pXest_type, model.ptr_pXest) + p6est_vertically_refine!(ptr_new_pXest, + refine_layer_fn_callback_c, + C_NULL) + ptr_new_pXest +end + +function vertically_uniformly_refine(model::OctreeDistributedDiscreteModel) + ptr_new_pXest = _vertically_uniformly_refine!(model) + + # Extract ghost and lnodes + ptr_pXest_ghost = setup_pXest_ghost(model.pXest_type, ptr_new_pXest) + ptr_pXest_lnodes = setup_pXest_lnodes_nonconforming(model.pXest_type, ptr_new_pXest, ptr_pXest_ghost) + + # Build fine-grid mesh + fmodel,non_conforming_glue = setup_non_conforming_distributed_discrete_model(model.pXest_type, + model.parts, + model.coarse_model, + model.ptr_pXest_connectivity, + ptr_new_pXest, + ptr_pXest_ghost, + ptr_pXest_lnodes) + + pXest_ghost_destroy(model.pXest_type,ptr_pXest_ghost) + pXest_lnodes_destroy(model.pXest_type,ptr_pXest_lnodes) + + pXest_refinement_rule_type = PXestVerticalRefinementRuleType() + _refinement_and_coarsening_flags = map(partition(get_cell_gids(model))) do indices + flags = Vector{Cint}(undef,length(local_to_global(indices))) + flags .= refine_flag + end + + stride = pXest_stride_among_children(model.pXest_type,pXest_refinement_rule_type,model.ptr_pXest) + adaptivity_glue = _compute_fine_to_coarse_model_glue(model.pXest_type, + pXest_refinement_rule_type, + model.parts, + model.dmodel, + fmodel, + _refinement_and_coarsening_flags, + stride) + adaptive_models = map(local_views(model), + local_views(fmodel), + adaptivity_glue) do model, fmodel, glue + Gridap.Adaptivity.AdaptedDiscreteModel(fmodel,model,glue) + end + fmodel = GridapDistributed.GenericDistributedDiscreteModel(adaptive_models,get_cell_gids(fmodel)) + ref_model = OctreeDistributedDiscreteModel(3,3, + model.parts, + fmodel, + non_conforming_glue, + model.coarse_model, + model.ptr_pXest_connectivity, + ptr_new_pXest, + model.pXest_type, + model.pXest_refinement_rule_type, + false, + model) + return ref_model, adaptivity_glue +end + + + function setup_non_conforming_distributed_discrete_model(pXest_type::P6estType, parts, coarse_discrete_model, diff --git a/src/GridapP4est.jl b/src/GridapP4est.jl index a56bcca..bcb6c15 100644 --- a/src/GridapP4est.jl +++ b/src/GridapP4est.jl @@ -26,6 +26,7 @@ module GridapP4est export AnisotropicallyAdapted3DDistributedDiscreteModel export vertically_adapt export horizontally_adapt + export vertically_uniformly_refine export nothing_flag, refine_flag, coarsen_flag export FixedFractionAdaptiveFlagsMarkingStrategy export update_adaptivity_flags! diff --git a/src/OctreeDistributedDiscreteModels.jl b/src/OctreeDistributedDiscreteModels.jl index 3f950bf..c839f10 100644 --- a/src/OctreeDistributedDiscreteModels.jl +++ b/src/OctreeDistributedDiscreteModels.jl @@ -241,7 +241,7 @@ function Init(a::OctreeDistributedDiscreteModel) end function Finalize(a::OctreeDistributedDiscreteModel) - # octree_distributed_discrete_model_free!(a) + octree_distributed_discrete_model_free!(a) return nothing end diff --git a/test/PoissonAnisotropicOctreeModelsTests.jl b/test/PoissonAnisotropicOctreeModelsTests.jl index 16d1c79..df0c5c0 100644 --- a/test/PoissonAnisotropicOctreeModelsTests.jl +++ b/test/PoissonAnisotropicOctreeModelsTests.jl @@ -245,6 +245,9 @@ module PoissonAnisotropicOctreeModelsTests rdmodel=dmodel for i=1:num_amr_steps rdmodel=test_transfer_ops_and_redistribute(ranks,rdmodel,order,T) + if i==1 + rdmodel,_=vertically_uniformly_refine(dmodel) + end end end @@ -271,11 +274,8 @@ module PoissonAnisotropicOctreeModelsTests end for perm in (1,2), order in (1,4), scalar_or_vector in (:vector,) test(ranks,perm,order,_field_type(Val{3}(),scalar_or_vector)) - end - for order=1:1, scalar_or_vector in (:scalar,:vector) - test_3d(ranks,order,_field_type(Val{3}(),scalar_or_vector), num_amr_steps=4) end - for order=1:1, scalar_or_vector in (:scalar,) + for order=1:1, scalar_or_vector in (:scalar,:vector) test_3d(ranks,order,_field_type(Val{3}(),scalar_or_vector), num_amr_steps=4) end end From a63f5f036e1dd41042fbc0d4039cd15476fe3a49 Mon Sep 17 00:00:00 2001 From: "Alberto F. Martin" Date: Sat, 9 Mar 2024 23:31:12 +1100 Subject: [PATCH 09/29] Deactivating temporarily 2D+1D AMR tests --- test/runtests.jl | 6 +++--- 1 file changed, 3 insertions(+), 3 deletions(-) diff --git a/test/runtests.jl b/test/runtests.jl index e8819a1..6c11412 100644 --- a/test/runtests.jl +++ b/test/runtests.jl @@ -47,9 +47,9 @@ function run_tests(testdir) elseif f in ["PoissonNonConformingOctreeModelsTests.jl"] np = [1,2,4] extra_args = "" - elseif f in ["PoissonAnisotropicOctreeModelsTests.jl"] - np = [1,4] - extra_args = "" + #elseif f in ["PoissonAnisotropicOctreeModelsTests.jl"] + # np = [1,4] + # extra_args = "" else np = [nprocs] extra_args = "" From b826ca5ecec83b4612604c923e80a8cf7652b437 Mon Sep 17 00:00:00 2001 From: "Alberto F. Martin" Date: Sun, 10 Mar 2024 12:58:49 +1100 Subject: [PATCH 10/29] Avoiding 2D+1D AMR tests in github actions --- test/runtests.jl | 7 ++++--- 1 file changed, 4 insertions(+), 3 deletions(-) diff --git a/test/runtests.jl b/test/runtests.jl index 6c11412..7b852ad 100644 --- a/test/runtests.jl +++ b/test/runtests.jl @@ -47,9 +47,10 @@ function run_tests(testdir) elseif f in ["PoissonNonConformingOctreeModelsTests.jl"] np = [1,2,4] extra_args = "" - #elseif f in ["PoissonAnisotropicOctreeModelsTests.jl"] - # np = [1,4] - # extra_args = "" + elseif f in ["PoissonAnisotropicOctreeModelsTests.jl"] + continue + np = [1,4] + extra_args = "" else np = [nprocs] extra_args = "" From 37306a285db56df6d0c65e1111386f6e0528863d Mon Sep 17 00:00:00 2001 From: "Alberto F. Martin" Date: Sun, 10 Mar 2024 13:07:10 +1100 Subject: [PATCH 11/29] fix --- test/runtests.jl | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/test/runtests.jl b/test/runtests.jl index 7b852ad..b28b5e3 100644 --- a/test/runtests.jl +++ b/test/runtests.jl @@ -48,7 +48,7 @@ function run_tests(testdir) np = [1,2,4] extra_args = "" elseif f in ["PoissonAnisotropicOctreeModelsTests.jl"] - continue + return np = [1,4] extra_args = "" else From 35ba2094d3a8e9437de5eea0dac538fcd06ea7a7 Mon Sep 17 00:00:00 2001 From: "Alberto F. Martin" Date: Mon, 11 Mar 2024 19:59:32 +1100 Subject: [PATCH 12/29] Activating Anistropic tests/Deactivating the rest --- test/runtests.jl | 7 +++++-- 1 file changed, 5 insertions(+), 2 deletions(-) diff --git a/test/runtests.jl b/test/runtests.jl index b28b5e3..d1e5f3d 100644 --- a/test/runtests.jl +++ b/test/runtests.jl @@ -34,21 +34,24 @@ function run_tests(testdir) @time @testset "$f" for f in testfiles MPI.mpiexec() do cmd if f in ["PoissonUniformlyRefinedOctreeModelsTests.jl"] + return np = [4] extra_args = "-s 2 2 -r 2" elseif f in ["OctreeDistributedDiscreteModelsTests.jl", "OctreeDistributedDiscreteModelsNoEnvTests.jl", "AdaptivityFlagsMarkingStrategiesTests.jl"] + return np = [4] extra_args = "" elseif f in ["DarcyNonConformingOctreeModelsTests.jl"] + return np = [1,4] extra_args = "" elseif f in ["PoissonNonConformingOctreeModelsTests.jl"] + return np = [1,2,4] extra_args = "" - elseif f in ["PoissonAnisotropicOctreeModelsTests.jl"] - return + elseif f in ["PoissonAnisotropicOctreeModelsTests.jl"] np = [1,4] extra_args = "" else From d1bf5657c4fbd1eb85bdaf56cfffd4bfa1579b24 Mon Sep 17 00:00:00 2001 From: "Alberto F. Martin" Date: Mon, 11 Mar 2024 20:11:21 +1100 Subject: [PATCH 13/29] Deactivating temporarily unsage_store! --- src/PXestTypeMethods.jl | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/src/PXestTypeMethods.jl b/src/PXestTypeMethods.jl index e15d595..ad4f3f9 100644 --- a/src/PXestTypeMethods.jl +++ b/src/PXestTypeMethods.jl @@ -498,7 +498,7 @@ function p6est_horizontally_adapt_reset_callbacks() q2_ptr=p2est_quadrant_array_index(forest.layers[], f) @assert p2est_quadrant_is_equal(q2_ptr,layer_ptr) - unsafe_store!(Ptr{Cint}(q2_ptr[].p.user_data), user_data, 1) + #unsafe_store!(Ptr{Cint}(q2_ptr[].p.user_data), user_data, 1) end current_layer_within_column=current_layer_within_column+1 From 065d08a6d031a8b8211d3e95f7726784e0f07868 Mon Sep 17 00:00:00 2001 From: "Alberto F. Martin" Date: Mon, 11 Mar 2024 20:27:45 +1100 Subject: [PATCH 14/29] Uncommenting reset Commenting 2D update flags --- src/OctreeDistributedDiscreteModels.jl | 3 ++- src/PXestTypeMethods.jl | 2 +- 2 files changed, 3 insertions(+), 2 deletions(-) diff --git a/src/OctreeDistributedDiscreteModels.jl b/src/OctreeDistributedDiscreteModels.jl index c839f10..29d87e7 100644 --- a/src/OctreeDistributedDiscreteModels.jl +++ b/src/OctreeDistributedDiscreteModels.jl @@ -347,7 +347,8 @@ function p6est_horizontally_adapt_update_flags!(ptr_pXest_old, ptr_pXest_new) flags=unsafe_wrap(Array, Ptr{Cint}(pXest_old.user_pointer), pXest_old.columns[].local_num_quadrants) - pXest_update_flags!(P4estType(), flags, ptr_p4est_old, ptr_p4est_new) + println("XXX: $(flags)") + #pXest_update_flags!(P4estType(), flags, ptr_p4est_old, ptr_p4est_new) end function p6est_vertically_adapt_update_flags!(ptr_pXest_old, ptr_pXest_new) diff --git a/src/PXestTypeMethods.jl b/src/PXestTypeMethods.jl index ad4f3f9..e15d595 100644 --- a/src/PXestTypeMethods.jl +++ b/src/PXestTypeMethods.jl @@ -498,7 +498,7 @@ function p6est_horizontally_adapt_reset_callbacks() q2_ptr=p2est_quadrant_array_index(forest.layers[], f) @assert p2est_quadrant_is_equal(q2_ptr,layer_ptr) - #unsafe_store!(Ptr{Cint}(q2_ptr[].p.user_data), user_data, 1) + unsafe_store!(Ptr{Cint}(q2_ptr[].p.user_data), user_data, 1) end current_layer_within_column=current_layer_within_column+1 From 2ec6f9e7c8c9c00440c0f8c1ed1fbcc4cd4835e6 Mon Sep 17 00:00:00 2001 From: "Alberto F. Martin" Date: Sat, 16 Mar 2024 22:33:08 +1100 Subject: [PATCH 15/29] * pXest_copy calls now copies user data. Not copying user data was causing double free or corruption. * No need to count local num quadrants in the horizontal for p6est --- ...callyAdapted3DDistributedDiscreteModels.jl | 23 +++++++++---------- src/PXestTypeMethods.jl | 6 ++--- 2 files changed, 14 insertions(+), 15 deletions(-) diff --git a/src/AnisotropicallyAdapted3DDistributedDiscreteModels.jl b/src/AnisotropicallyAdapted3DDistributedDiscreteModels.jl index 10177bc..0a738f5 100644 --- a/src/AnisotropicallyAdapted3DDistributedDiscreteModels.jl +++ b/src/AnisotropicallyAdapted3DDistributedDiscreteModels.jl @@ -69,9 +69,9 @@ function _vertically_refine_coarsen_balance!(model::OctreeDistributedDiscreteMod ptr_new_pXest end -function vertically_adapt(model::OctreeDistributedDiscreteModel{Dc,Dp}, +function vertically_adapt(model::OctreeDistributedDiscreteModel{3,3}, refinement_and_coarsening_flags::MPIArray{<:Vector{<:Integer}}; - parts=nothing) where {Dc,Dp} + parts=nothing) Gridap.Helpers.@notimplementedif parts!=nothing @@ -111,7 +111,7 @@ function vertically_adapt(model::OctreeDistributedDiscreteModel{Dc,Dp}, Gridap.Adaptivity.AdaptedDiscreteModel(fmodel,model,glue) end fmodel = GridapDistributed.GenericDistributedDiscreteModel(adaptive_models,get_cell_gids(fmodel)) - ref_model = OctreeDistributedDiscreteModel(Dc,Dp, + ref_model = OctreeDistributedDiscreteModel(3,3, model.parts, fmodel, non_conforming_glue, @@ -578,13 +578,7 @@ function num_locally_owned_columns(octree_model) @assert octree_model.pXest_type==P6estType() map(octree_model.parts) do _ pXest=octree_model.ptr_pXest[] - num_cols = 0 - num_trees = Cint(pXest.columns[].connectivity[].num_trees) - for itree = 0:num_trees-1 - tree = pXest_tree_array_index(octree_model.pXest_type,pXest,itree)[] - num_cols += tree.quadrants.elem_count - end - num_cols + pXest.columns[].local_num_quadrants end end @@ -652,15 +646,20 @@ function horizontally_adapt(model::OctreeDistributedDiscreteModel{Dc,Dp}, pXest_lnodes_destroy(model.pXest_type,ptr_pXest_lnodes) pXest_refinement_rule_type = PXestHorizontalRefinementRuleType() - extruded_ref_coarsen_flags=map(partition(get_cell_gids(model)),refinement_and_coarsening_flags) do indices, flags + + extruded_ref_coarsen_flags=map(partition(get_cell_gids(model)),_refinement_and_coarsening_flags) do indices, flags similar(flags, length(local_to_global(indices))) end + + _extrude_refinement_and_coarsening_flags!(extruded_ref_coarsen_flags, - refinement_and_coarsening_flags, + _refinement_and_coarsening_flags, model.ptr_pXest, ptr_new_pXest) + + stride = pXest_stride_among_children(model.pXest_type,pXest_refinement_rule_type,model.ptr_pXest) adaptivity_glue = _compute_fine_to_coarse_model_glue(model.pXest_type, diff --git a/src/PXestTypeMethods.jl b/src/PXestTypeMethods.jl index e15d595..9028a3a 100644 --- a/src/PXestTypeMethods.jl +++ b/src/PXestTypeMethods.jl @@ -269,15 +269,15 @@ function pXest_coarsen!(::P8estType, ptr_pXest, coarsen_fn_c) end function pXest_copy(::P4estType, ptr_pXest) - p4est_copy(ptr_pXest, Cint(0)) + p4est_copy(ptr_pXest, Cint(1)) end function pXest_copy(::P6estType, ptr_pXest) - p6est_copy(ptr_pXest, Cint(0)) + p6est_copy(ptr_pXest, Cint(1)) end function pXest_copy(::P8estType, ptr_pXest) - p8est_copy(ptr_pXest, Cint(0)) + p8est_copy(ptr_pXest, Cint(1)) end function pXest_balance!(::P4estType, ptr_pXest; k_2_1_balance=0) From a5a29ca273532b3323838ac15baf2e7534cda8af Mon Sep 17 00:00:00 2001 From: JordiManyer Date: Fri, 22 Mar 2024 10:42:11 +1100 Subject: [PATCH 16/29] Updated compats --- Project.toml | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/Project.toml b/Project.toml index 08108fe..3f37bb5 100644 --- a/Project.toml +++ b/Project.toml @@ -17,8 +17,8 @@ Test = "8dfed614-e22c-5e08-85e1-65c5234f0b40" [compat] ArgParse = "1" FillArrays = "0.8.4, 0.9, 0.10, 0.11, 0.12, 1" -Gridap = "0.17.22" -GridapDistributed = "0.3.1" +Gridap = "0.17.22, 0.18" +GridapDistributed = "0.3.1, 0.4" MPI = "0.20" P4est_wrapper = "0.2.0" PartitionedArrays = "0.3.3" From 658a71d9701c858baacc0fb5e1d05c99ac0a2799 Mon Sep 17 00:00:00 2001 From: JordiManyer Date: Fri, 22 Mar 2024 11:29:15 +1100 Subject: [PATCH 17/29] Bumped version to 0.3.6 --- Project.toml | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/Project.toml b/Project.toml index 3f37bb5..ab30aff 100644 --- a/Project.toml +++ b/Project.toml @@ -1,7 +1,7 @@ name = "GridapP4est" uuid = "c2c8e14b-f5fd-423d-9666-1dd9ad120af9" authors = ["Alberto F. Martin "] -version = "0.3.5" +version = "0.3.6" [deps] ArgParse = "c7e460c6-2fb9-53a9-8c5b-16f535851c63" From a28c9a77037beebd1f339a1542229e9194e72d74 Mon Sep 17 00:00:00 2001 From: JordiManyer Date: Sun, 24 Mar 2024 16:59:52 +1100 Subject: [PATCH 18/29] Small bugfix --- src/FESpaces.jl | 5 +++-- 1 file changed, 3 insertions(+), 2 deletions(-) diff --git a/src/FESpaces.jl b/src/FESpaces.jl index 35c7747..2b0f9c8 100644 --- a/src/FESpaces.jl +++ b/src/FESpaces.jl @@ -935,9 +935,10 @@ function _add_constraints(model::GridapDistributed.DistributedDiscreteModel{Dc}, map(partition(gids)) do indices @debug "[$(part_id(indices))]: l2g_cell_gids=$(local_to_global(indices))" @debug "[$(part_id(indices))]: l2o_owner=$(local_to_owner(indices))" - end + end + trian = Triangulation(model) vector_type = GridapDistributed._find_vector_type(spaces_w_constraints,gids) - GridapDistributed.DistributedSingleFieldFESpace(spaces_w_constraints,gids,vector_type) + GridapDistributed.DistributedSingleFieldFESpace(spaces_w_constraints,gids,trian,vector_type) end # Generates a new DistributedSingleFieldFESpace composed From 202413f7a41b4f88e30cb9e3196fff5b7dd890bf Mon Sep 17 00:00:00 2001 From: JordiManyer Date: Sun, 24 Mar 2024 17:04:01 +1100 Subject: [PATCH 19/29] Minor --- src/FESpaces.jl | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/src/FESpaces.jl b/src/FESpaces.jl index 2b0f9c8..9a90dc6 100644 --- a/src/FESpaces.jl +++ b/src/FESpaces.jl @@ -936,7 +936,7 @@ function _add_constraints(model::GridapDistributed.DistributedDiscreteModel{Dc}, @debug "[$(part_id(indices))]: l2g_cell_gids=$(local_to_global(indices))" @debug "[$(part_id(indices))]: l2o_owner=$(local_to_owner(indices))" end - trian = Triangulation(model) + trian = GridapDistributed.DistributedTriangulation(map(get_triangulation,spaces_w_constraints),model) vector_type = GridapDistributed._find_vector_type(spaces_w_constraints,gids) GridapDistributed.DistributedSingleFieldFESpace(spaces_w_constraints,gids,trian,vector_type) end From ec89e6cd396f73a359c708c5acffbd987a5899ee Mon Sep 17 00:00:00 2001 From: "Alberto F. Martin" Date: Sun, 24 Mar 2024 18:32:57 +1100 Subject: [PATCH 20/29] Uncommented p4est flags update --- ...callyAdapted3DDistributedDiscreteModels.jl | 1 + src/OctreeDistributedDiscreteModels.jl | 21 +++++++++++++++++-- test/PoissonAnisotropicOctreeModelsTests.jl | 8 +------ 3 files changed, 21 insertions(+), 9 deletions(-) diff --git a/src/AnisotropicallyAdapted3DDistributedDiscreteModels.jl b/src/AnisotropicallyAdapted3DDistributedDiscreteModels.jl index 0a738f5..9dd88a3 100644 --- a/src/AnisotropicallyAdapted3DDistributedDiscreteModels.jl +++ b/src/AnisotropicallyAdapted3DDistributedDiscreteModels.jl @@ -624,6 +624,7 @@ function horizontally_adapt(model::OctreeDistributedDiscreteModel{Dc,Dp}, Gridap.Helpers.@notimplementedif parts!=nothing _refinement_and_coarsening_flags = map(refinement_and_coarsening_flags) do flags + @debug "[$(MPI.Comm_rank(MPI.COMM_WORLD))] horizontally_adapt_flags=$(flags)" convert(Vector{Cint},flags) end diff --git a/src/OctreeDistributedDiscreteModels.jl b/src/OctreeDistributedDiscreteModels.jl index 29d87e7..3ef2e4b 100644 --- a/src/OctreeDistributedDiscreteModels.jl +++ b/src/OctreeDistributedDiscreteModels.jl @@ -347,8 +347,7 @@ function p6est_horizontally_adapt_update_flags!(ptr_pXest_old, ptr_pXest_new) flags=unsafe_wrap(Array, Ptr{Cint}(pXest_old.user_pointer), pXest_old.columns[].local_num_quadrants) - println("XXX: $(flags)") - #pXest_update_flags!(P4estType(), flags, ptr_p4est_old, ptr_p4est_new) + pXest_update_flags!(P4estType(), flags, ptr_p4est_old, ptr_p4est_new) end function p6est_vertically_adapt_update_flags!(ptr_pXest_old, ptr_pXest_new) @@ -432,6 +431,8 @@ function _compute_fine_to_coarse_model_glue( # Note: Reversing snd and rcv lids_rcv,lids_snd = map(PArrays.getany,assembly_local_indices(partition(fgids))) + @debug "$(MPI.Comm_rank(MPI.COMM_WORLD)): lids_rcv: $(lids_rcv)" + @debug "$(MPI.Comm_rank(MPI.COMM_WORLD)): lids_snd: $(lids_snd)" cgids_data = local_to_global(cpartition)[fine_to_coarse_faces_map[Dc+1][lids_snd.data]] cgids_snd = PArrays.JaggedArray(cgids_data,lids_snd.ptrs) cgids_rcv = PArrays.JaggedArray(Vector{Int}(undef,length(lids_rcv.data)),lids_rcv.ptrs) @@ -598,6 +599,8 @@ function _compute_fine_to_coarse_model_glue( cpartition::AbstractLocalIndices) # Note: Reversing snd and rcv lids_rcv,lids_snd = map(PArrays.getany,assembly_local_indices(fpartition)) + @debug "$(MPI.Comm_rank(MPI.COMM_WORLD)): lids_rcv: $(lids_rcv)" + @debug "$(MPI.Comm_rank(MPI.COMM_WORLD)): lids_snd: $(lids_snd)" cgids_data = local_to_global(cpartition)[fine_to_coarse_faces_map[end][lids_snd.data]] cgids_snd = PArrays.JaggedArray(cgids_data,lids_snd.ptrs) cgids_rcv = PArrays.JaggedArray(Vector{Int}(undef,length(lids_rcv.data)),lids_rcv.ptrs) @@ -814,6 +817,7 @@ function _compute_fine_to_coarse_model_glue( gid = cgids_rcv.data[i] fine_to_coarse_faces_map[end][lid] = glo_to_loc[gid] fcell_to_child_id[lid] = child_id_rcv.data[i] + @debug "[$(MPI.Comm_rank(MPI.COMM_WORLD))] fcell_to_child_id.data[$(lid)]=$(child_id_rcv.data[i])" end end end @@ -848,6 +852,7 @@ function _compute_fine_to_coarse_model_glue( @assert (ptrs[lid+1]-ptrs[lid])==1 data[ptrs[lid]] = glo_to_loc[gid] fcell_to_child_id.data[ptrs[lid]] = child_id_rcv.data[i] + @debug "[$(MPI.Comm_rank(MPI.COMM_WORLD))] fcell_to_child_id.data[$(ptrs[lid])]=$(child_id_rcv.data[i])" end end end @@ -1075,6 +1080,7 @@ function _process_owned_cells_fine_to_coarse_model_glue(pXest_type, # Fill children data: fine_to_coarse_faces_map_data = Vector{Int}(undef,fine_to_coarse_faces_map_ptrs[end]-1) fcell_to_child_id_data = Vector{Int}(undef,fine_to_coarse_faces_map_ptrs[end]-1) + fcell_to_child_id_data .= -1 cell = 1 c = 1 while cell <= num_o_c_cells @@ -1121,6 +1127,13 @@ function _process_owned_cells_fine_to_coarse_model_glue(pXest_type, flags, num_o_c_cells, stride) + + @debug "[$(MPI.Comm_rank(MPI.COMM_WORLD))] num_o_c_cells: $(num_o_c_cells)" + @debug "[$(MPI.Comm_rank(MPI.COMM_WORLD))] flags: $(flags)" + @debug "[$(MPI.Comm_rank(MPI.COMM_WORLD))] len(fcell_to_child_id): $(length(fcell_to_child_id))" + @debug "[$(MPI.Comm_rank(MPI.COMM_WORLD))] stride: $(stride)" + + # Go over all cells of coarse grid portion num_children = get_num_children(pXest_type,pXest_refinement_rule_type) c = 1 @@ -1132,6 +1145,7 @@ function _process_owned_cells_fine_to_coarse_model_glue(pXest_type, for j=1:stride fine_to_coarse_faces_map[c] = current_cell fcell_to_child_id[c] = child + @debug "[$(MPI.Comm_rank(MPI.COMM_WORLD))] fcell_to_child_id[$(c)]: $(child)" c+=1 current_cell+=1 end @@ -1140,6 +1154,7 @@ function _process_owned_cells_fine_to_coarse_model_glue(pXest_type, elseif (flags[cell]==nothing_flag) fine_to_coarse_faces_map[c] = cell fcell_to_child_id[c] = 1 + @debug "[$(MPI.Comm_rank(MPI.COMM_WORLD))] fcell_to_child_id[$(c)]: 1" c+=1 cell+=1 else @@ -1151,6 +1166,7 @@ function _process_owned_cells_fine_to_coarse_model_glue(pXest_type, num_f_cells = num_cells(fmodel) # Number of fine cells (owned+ghost) num_o_c_cells = own_length(cpartition) # Number of coarse cells (owned) + @debug "[$(MPI.Comm_rank(MPI.COMM_WORLD))] own_length(fpartition): $(own_length(fpartition))" ftopology = Gridap.Geometry.get_grid_topology(fmodel) if coarsen fine_to_coarse_faces_map = Vector{Gridap.Arrays.Table{Int,Vector{Int},Vector{Int}}}(undef,Dc+1) @@ -1163,6 +1179,7 @@ function _process_owned_cells_fine_to_coarse_model_glue(pXest_type, fine_to_coarse_faces_map = Vector{Vector{Int}}(undef,Dc+1) fine_to_coarse_faces_map[Dc+1] = Vector{Int}(undef,num_f_cells) fcell_to_child_id = Vector{Int}(undef,num_f_cells) + fcell_to_child_id .= -1 _setup_fine_to_coarse_faces_map_vector!(pXest_type, pXest_refinement_rule_type, fine_to_coarse_faces_map[Dc+1], diff --git a/test/PoissonAnisotropicOctreeModelsTests.jl b/test/PoissonAnisotropicOctreeModelsTests.jl index df0c5c0..87fc3f8 100644 --- a/test/PoissonAnisotropicOctreeModelsTests.jl +++ b/test/PoissonAnisotropicOctreeModelsTests.jl @@ -34,15 +34,11 @@ module PoissonAnisotropicOctreeModelsTests VH=FESpace(dmodel,reffe;dirichlet_tags="boundary") UH=TrialFESpace(VH,u) num_local_cols=GridapP4est.num_locally_owned_columns(dmodel) - ref_coarse_flags=map(ranks,num_local_cols) do rank,num_local_cols - println("[rank:$(rank)] $(num_local_cols)") - + ref_coarse_flags=map(ranks,num_local_cols) do rank,num_local_cols flags=zeros(Cint,num_local_cols) flags.=nothing_flag - flags[1]=refine_flag flags[end]=refine_flag - # To create some unbalance if (rank%2==0) flags[div(num_local_cols,2)]=refine_flag @@ -51,8 +47,6 @@ module PoissonAnisotropicOctreeModelsTests end fmodel,glue=GridapP4est.horizontally_adapt(dmodel,ref_coarse_flags); - writevtk(fmodel,"fmodel") - Vh=FESpace(fmodel,reffe,conformity=:H1;dirichlet_tags="boundary") Uh=TrialFESpace(Vh,u) From 8fae1af069158929b56bb89d3ceb0f2bf23e90cf Mon Sep 17 00:00:00 2001 From: "Alberto F. Martin" Date: Tue, 26 Mar 2024 19:26:53 +1100 Subject: [PATCH 21/29] Coarsening seems to be working with 2D+1D AMR --- ...callyAdapted3DDistributedDiscreteModels.jl | 1 + src/OctreeDistributedDiscreteModels.jl | 145 +++++++++++------- test/PoissonAnisotropicOctreeModelsTests.jl | 14 +- 3 files changed, 94 insertions(+), 66 deletions(-) diff --git a/src/AnisotropicallyAdapted3DDistributedDiscreteModels.jl b/src/AnisotropicallyAdapted3DDistributedDiscreteModels.jl index 9dd88a3..8f7d6a1 100644 --- a/src/AnisotropicallyAdapted3DDistributedDiscreteModels.jl +++ b/src/AnisotropicallyAdapted3DDistributedDiscreteModels.jl @@ -625,6 +625,7 @@ function horizontally_adapt(model::OctreeDistributedDiscreteModel{Dc,Dp}, _refinement_and_coarsening_flags = map(refinement_and_coarsening_flags) do flags @debug "[$(MPI.Comm_rank(MPI.COMM_WORLD))] horizontally_adapt_flags=$(flags)" + println(flags) convert(Vector{Cint},flags) end diff --git a/src/OctreeDistributedDiscreteModels.jl b/src/OctreeDistributedDiscreteModels.jl index 3ef2e4b..9076a09 100644 --- a/src/OctreeDistributedDiscreteModels.jl +++ b/src/OctreeDistributedDiscreteModels.jl @@ -551,13 +551,13 @@ function _process_owned_cells_fine_to_coarse_model_glue(cmodel::DiscreteModel{Dc return fine_to_coarse_faces_map, fine_to_coarse_faces_dim, fcell_to_child_id end -function _move_fwd_and_check_if_all_children_coarsened(flags,num_o_c_cells,cell,num_children) - e=cell+num_children-1 +function _move_fwd_and_check_if_all_children_coarsened(flags,num_o_c_cells,cell,num_children,stride) + e=cell+num_children*stride-1 while (cell <= num_o_c_cells) && (cell <= e) if (flags[cell]!=coarsen_flag) break end - cell = cell+1 + cell = cell+stride end return cell,cell==e+1 end @@ -707,13 +707,14 @@ function _compute_fine_to_coarse_model_glue( function _check_if_coarsen(pXest_type, pXest_refinement_rule_type, cpartition, - flags) + flags, + stride) num_children = get_num_children(pXest_type,pXest_refinement_rule_type) num_o_c_cells = own_length(cpartition) cell = 1 while cell <= num_o_c_cells if (flags[cell]==coarsen_flag) - cell,coarsen = _move_fwd_and_check_if_all_children_coarsened(flags,num_o_c_cells,cell,num_children) + cell,coarsen = _move_fwd_and_check_if_all_children_coarsened(flags,num_o_c_cells,cell,num_children,stride) if coarsen return true end @@ -911,7 +912,7 @@ function _compute_fine_to_coarse_model_glue( # Check if there is at least one part in which a cell was coarsened cgids = get_cell_gids(cmodel) coarsen_array = map(partition(cgids),refinement_and_coarsening_flags) do cpartition,flags - _check_if_coarsen(pXest_type,pXest_refinement_rule_type,cpartition,flags) + _check_if_coarsen(pXest_type,pXest_refinement_rule_type,cpartition,flags,stride) end or_func(a,b)=a || b coarsen = PArrays.getany(reduction(or_func,coarsen_array;destination=:all,init=false)) @@ -1034,46 +1035,56 @@ function _process_owned_cells_fine_to_coarse_model_glue(pXest_type, coarsen, stride) where Dc - function _setup_fine_to_coarse_faces_map_table(pXest_type,pXest_refinement_rule_type,flags,num_o_c_cells,num_f_cells) + function _setup_fine_to_coarse_faces_map_table(pXest_type, + pXest_refinement_rule_type, + flags, + num_o_c_cells, + num_f_cells, + stride) num_children = get_num_children(pXest_type, pXest_refinement_rule_type) # Count cell children: fine_to_coarse_faces_map_ptrs = Vector{Int}(undef,num_f_cells+1) fine_to_coarse_faces_map_ptrs[1] = 1 - cell = 1 - c = 1 - while cell <= num_o_c_cells # For each coarse cell - if (flags[cell]==nothing_flag) # Cell not touched - fine_to_coarse_faces_map_ptrs[c+1] = fine_to_coarse_faces_map_ptrs[c]+1 - cell = cell+1 - c = c+1 - elseif (flags[cell]==refine_flag) # Cell is refined + old_cell = 1 + new_cell = 1 + while old_cell <= num_o_c_cells # For each coarse cell + if (flags[old_cell]==nothing_flag) # Cell not touched + fine_to_coarse_faces_map_ptrs[new_cell+1] = fine_to_coarse_faces_map_ptrs[new_cell]+1 + old_cell += 1 + new_cell += 1 + elseif (flags[old_cell]==refine_flag) # Cell is refined for child = 1:num_children - fine_to_coarse_faces_map_ptrs[c+1] = fine_to_coarse_faces_map_ptrs[c]+1 - c = c+1 + for j=1:stride + fine_to_coarse_faces_map_ptrs[new_cell+1] = fine_to_coarse_faces_map_ptrs[new_cell]+1 + new_cell+=1 + end end - cell = cell+1 + old_cell=old_cell+stride else # Cell is coarsened - @assert flags[cell]==coarsen_flag - cell_fwd,coarsen = _move_fwd_and_check_if_all_children_coarsened(flags,num_o_c_cells,cell,num_children) + @assert flags[old_cell]==coarsen_flag + cell_fwd,coarsen = _move_fwd_and_check_if_all_children_coarsened(flags,num_o_c_cells,old_cell,num_children,stride) if coarsen - @assert cell_fwd-cell==num_children - fine_to_coarse_faces_map_ptrs[c+1] = fine_to_coarse_faces_map_ptrs[c]+num_children - cell = cell+num_children - c = c+1 - else - for j = c:c+(cell_fwd-cell+1) - fine_to_coarse_faces_map_ptrs[j+1] = fine_to_coarse_faces_map_ptrs[j]+1 - c = c+1 - end - cell = cell_fwd - c = c+cell_fwd-cell + @assert (cell_fwd-old_cell)==(num_children*stride) + for j=1:stride + fine_to_coarse_faces_map_ptrs[new_cell+1] = fine_to_coarse_faces_map_ptrs[new_cell]+num_children + old_cell = old_cell+num_children + new_cell = new_cell+1 + end + else + @Gridap.Helpers.@unreachable + # for j = new_cell:new_cell+(cell_fwd-old_cell+1) + # fine_to_coarse_faces_map_ptrs[j+1] = fine_to_coarse_faces_map_ptrs[j]+1 + # new_cell = new_cell+1 + # end + # old_cell = cell_fwd + # new_cell = new_cell+cell_fwd-old_cell end end end # Counts to pointers: - for j = c:num_f_cells + for j = new_cell:num_f_cells fine_to_coarse_faces_map_ptrs[j+1] = fine_to_coarse_faces_map_ptrs[j] end @@ -1081,38 +1092,47 @@ function _process_owned_cells_fine_to_coarse_model_glue(pXest_type, fine_to_coarse_faces_map_data = Vector{Int}(undef,fine_to_coarse_faces_map_ptrs[end]-1) fcell_to_child_id_data = Vector{Int}(undef,fine_to_coarse_faces_map_ptrs[end]-1) fcell_to_child_id_data .= -1 - cell = 1 - c = 1 - while cell <= num_o_c_cells - if (flags[cell]==refine_flag) # Cell is refined + old_cell = 1 + new_cell = 1 + while old_cell <= num_o_c_cells + if (flags[old_cell]==refine_flag) # Cell is refined for child = 1:num_children - fine_to_coarse_faces_map_data[fine_to_coarse_faces_map_ptrs[c]+child-1] = cell - fcell_to_child_id_data[fine_to_coarse_faces_map_ptrs[c]+child-1] = child - end - c = c + num_children - cell = cell+1 - elseif (flags[cell]==nothing_flag) # Cell is not touched - fine_to_coarse_faces_map_data[fine_to_coarse_faces_map_ptrs[c]]=cell - fcell_to_child_id_data[fine_to_coarse_faces_map_ptrs[c]]=1 - c = c+1 - cell = cell+1 + current_cell=old_cell + for j=1:stride + fine_to_coarse_faces_map_data[fine_to_coarse_faces_map_ptrs[new_cell]] = current_cell + fcell_to_child_id_data[fine_to_coarse_faces_map_ptrs[new_cell]] = child + new_cell+=1 + current_cell+=1 + end + end + old_cell=old_cell+stride + elseif (flags[old_cell]==nothing_flag) # Cell is not touched + fine_to_coarse_faces_map_data[fine_to_coarse_faces_map_ptrs[new_cell]]=old_cell + fcell_to_child_id_data[fine_to_coarse_faces_map_ptrs[new_cell]]=1 + new_cell+=1 + old_cell+=1 else # Cell is coarsened - @assert flags[cell]==coarsen_flag - cell_fwd,coarsen=_move_fwd_and_check_if_all_children_coarsened(flags,num_o_c_cells,cell,num_children) + @assert flags[old_cell]==coarsen_flag + cell_fwd,coarsen=_move_fwd_and_check_if_all_children_coarsened(flags,num_o_c_cells,old_cell,num_children,stride) if coarsen for child = 1:num_children - fcell_to_child_id_data[fine_to_coarse_faces_map_ptrs[c]+child-1] = child - fine_to_coarse_faces_map_data[fine_to_coarse_faces_map_ptrs[c]+child-1] = cell - cell=cell+1 + current_cell=new_cell + for j=1:stride + fine_to_coarse_faces_map_data[fine_to_coarse_faces_map_ptrs[current_cell]+child-1] = old_cell + fcell_to_child_id_data[fine_to_coarse_faces_map_ptrs[current_cell]+child-1] = child + old_cell+=1 + current_cell+=1 + end end - c = c+1 + new_cell=new_cell+stride else - for j = cell:cell_fwd-1 - fine_to_coarse_faces_map_data[fine_to_coarse_faces_map_ptrs[c]]=j - fcell_to_child_id_data[fine_to_coarse_faces_map_ptrs[c]]=1 - c = c+1 - cell = cell+1 - end + @Gridap.Helpers.@unreachable + # for j = cell:cell_fwd-1 + # fine_to_coarse_faces_map_data[fine_to_coarse_faces_map_ptrs[c]]=j + # fcell_to_child_id_data[fine_to_coarse_faces_map_ptrs[c]]=1 + # c = c+1 + # cell = cell+1 + # end end end end @@ -1157,6 +1177,8 @@ function _process_owned_cells_fine_to_coarse_model_glue(pXest_type, @debug "[$(MPI.Comm_rank(MPI.COMM_WORLD))] fcell_to_child_id[$(c)]: 1" c+=1 cell+=1 + elseif (flags[cell]==coarsen_flag) + Gridap.Helpers.@notimplemented else @assert flags[cell]!=coarsen_flag error("Unknown AMR flag") @@ -1170,7 +1192,12 @@ function _process_owned_cells_fine_to_coarse_model_glue(pXest_type, ftopology = Gridap.Geometry.get_grid_topology(fmodel) if coarsen fine_to_coarse_faces_map = Vector{Gridap.Arrays.Table{Int,Vector{Int},Vector{Int}}}(undef,Dc+1) - a,b = _setup_fine_to_coarse_faces_map_table(pXest_type,pXest_refinement_rule_type,flags,num_o_c_cells,num_f_cells) + a,b = _setup_fine_to_coarse_faces_map_table(pXest_type, + pXest_refinement_rule_type, + flags, + num_o_c_cells, + num_f_cells, + stride) fine_to_coarse_faces_map[Dc+1] = a fcell_to_child_id = b # In the future we should also have here the code to also setup diff --git a/test/PoissonAnisotropicOctreeModelsTests.jl b/test/PoissonAnisotropicOctreeModelsTests.jl index 87fc3f8..12c49ca 100644 --- a/test/PoissonAnisotropicOctreeModelsTests.jl +++ b/test/PoissonAnisotropicOctreeModelsTests.jl @@ -170,11 +170,11 @@ module PoissonAnisotropicOctreeModelsTests ref_coarse_flags=map(ranks,num_local_cols) do rank,num_local_cols flags=zeros(Int,num_local_cols) flags.=nothing_flag - #if (rank==1) - # flags[1:4].=coarsen_flag - #else - flags[1]=refine_flag - #end + if (rank==1) + flags[1:4].=coarsen_flag + flags[5:6].=coarsen_flag + end + flags[end]=refine_flag flags end fmodel,glue=GridapP4est.horizontally_adapt(dmodel,ref_coarse_flags); @@ -235,7 +235,7 @@ module PoissonAnisotropicOctreeModelsTests coarse_model=CartesianDiscreteModel((0,1,0,1),(1,1)) dmodel=AnisotropicallyAdapted3DDistributedDiscreteModel(ranks, coarse_model, 2, 2) #writevtk(dmodel.dmodel,"model") - #test_refine_and_coarsen_at_once(ranks,dmodel,order,T) + test_refine_and_coarsen_at_once(ranks,dmodel,order,T) rdmodel=dmodel for i=1:num_amr_steps rdmodel=test_transfer_ops_and_redistribute(ranks,rdmodel,order,T) @@ -269,7 +269,7 @@ module PoissonAnisotropicOctreeModelsTests for perm in (1,2), order in (1,4), scalar_or_vector in (:vector,) test(ranks,perm,order,_field_type(Val{3}(),scalar_or_vector)) end - for order=1:1, scalar_or_vector in (:scalar,:vector) + for order=1:1, scalar_or_vector in (:scalar,) # (:scalar,:vector) test_3d(ranks,order,_field_type(Val{3}(),scalar_or_vector), num_amr_steps=4) end end From c00608e9a7f15a3f9c89894ac5fb0132e6df9d3f Mon Sep 17 00:00:00 2001 From: "Alberto F. Martin" Date: Wed, 27 Mar 2024 12:52:09 +1100 Subject: [PATCH 22/29] Typo --- src/OctreeDistributedDiscreteModels.jl | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/src/OctreeDistributedDiscreteModels.jl b/src/OctreeDistributedDiscreteModels.jl index 9076a09..f87d77f 100644 --- a/src/OctreeDistributedDiscreteModels.jl +++ b/src/OctreeDistributedDiscreteModels.jl @@ -1072,7 +1072,7 @@ function _process_owned_cells_fine_to_coarse_model_glue(pXest_type, new_cell = new_cell+1 end else - @Gridap.Helpers.@unreachable + Gridap.Helpers.@unreachable # for j = new_cell:new_cell+(cell_fwd-old_cell+1) # fine_to_coarse_faces_map_ptrs[j+1] = fine_to_coarse_faces_map_ptrs[j]+1 # new_cell = new_cell+1 @@ -1126,7 +1126,7 @@ function _process_owned_cells_fine_to_coarse_model_glue(pXest_type, end new_cell=new_cell+stride else - @Gridap.Helpers.@unreachable + Gridap.Helpers.@unreachable # for j = cell:cell_fwd-1 # fine_to_coarse_faces_map_data[fine_to_coarse_faces_map_ptrs[c]]=j # fcell_to_child_id_data[fine_to_coarse_faces_map_ptrs[c]]=1 From bc45c27915cb8caa07d02c8351204c2994b1f942 Mon Sep 17 00:00:00 2001 From: "Alberto F. Martin" Date: Wed, 27 Mar 2024 14:01:32 +1100 Subject: [PATCH 23/29] Minor --- test/PoissonAnisotropicOctreeModelsTests.jl | 1 - 1 file changed, 1 deletion(-) diff --git a/test/PoissonAnisotropicOctreeModelsTests.jl b/test/PoissonAnisotropicOctreeModelsTests.jl index 12c49ca..a9bdd89 100644 --- a/test/PoissonAnisotropicOctreeModelsTests.jl +++ b/test/PoissonAnisotropicOctreeModelsTests.jl @@ -172,7 +172,6 @@ module PoissonAnisotropicOctreeModelsTests flags.=nothing_flag if (rank==1) flags[1:4].=coarsen_flag - flags[5:6].=coarsen_flag end flags[end]=refine_flag flags From 03a4228cc388e9acea30ea8bd2be9e3c33c87745 Mon Sep 17 00:00:00 2001 From: "Alberto F. Martin" Date: Wed, 27 Mar 2024 15:04:18 +1100 Subject: [PATCH 24/29] Reactivating all tests --- test/runtests.jl | 4 ---- 1 file changed, 4 deletions(-) diff --git a/test/runtests.jl b/test/runtests.jl index d1e5f3d..ca5d975 100644 --- a/test/runtests.jl +++ b/test/runtests.jl @@ -34,21 +34,17 @@ function run_tests(testdir) @time @testset "$f" for f in testfiles MPI.mpiexec() do cmd if f in ["PoissonUniformlyRefinedOctreeModelsTests.jl"] - return np = [4] extra_args = "-s 2 2 -r 2" elseif f in ["OctreeDistributedDiscreteModelsTests.jl", "OctreeDistributedDiscreteModelsNoEnvTests.jl", "AdaptivityFlagsMarkingStrategiesTests.jl"] - return np = [4] extra_args = "" elseif f in ["DarcyNonConformingOctreeModelsTests.jl"] - return np = [1,4] extra_args = "" elseif f in ["PoissonNonConformingOctreeModelsTests.jl"] - return np = [1,2,4] extra_args = "" elseif f in ["PoissonAnisotropicOctreeModelsTests.jl"] From 9d5208ebb93cb906914e66882696b6aa7f963ab5 Mon Sep 17 00:00:00 2001 From: "Alberto F. Martin" Date: Thu, 28 Mar 2024 16:45:45 +1100 Subject: [PATCH 25/29] Freezing by now to P4est wrapper 0.2.1 --- Project.toml | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/Project.toml b/Project.toml index 800011d..63ce1bf 100644 --- a/Project.toml +++ b/Project.toml @@ -19,7 +19,7 @@ ArgParse = "1" FillArrays = "0.8.4, 0.9, 0.10, 0.11, 0.12, 1" Gridap = "0.17.22" MPI = "0.20" -P4est_wrapper = "0.2.1" +P4est_wrapper = "=0.2.1" PartitionedArrays = "0.3.3" julia = "1.5,1.6,1.7,1.8,1.9" From 281c4aa8f43219182d91314c76d57a5a7f50c05d Mon Sep 17 00:00:00 2001 From: "Alberto F. Martin" Date: Thu, 28 Mar 2024 17:27:05 +1100 Subject: [PATCH 26/29] typo fix in runtests.jl --- test/runtests.jl | 1 + 1 file changed, 1 insertion(+) diff --git a/test/runtests.jl b/test/runtests.jl index f805970..28168c1 100644 --- a/test/runtests.jl +++ b/test/runtests.jl @@ -49,6 +49,7 @@ function run_tests(testdir) extra_args = "" elseif f in ["PoissonAnisotropicOctreeModelsTests.jl"] np = [1,4] + extra_args = "" elseif f in ["PeriodicModels.jl"] np = [1] extra_args = "" From 2f4446f94182e37337fb1b309ebd0aea74f804ff Mon Sep 17 00:00:00 2001 From: "Alberto F. Martin" Date: Thu, 28 Mar 2024 17:46:40 +1100 Subject: [PATCH 27/29] Upgrading p4est to 2.3 --- .github/workflows/ci.yml | 6 +++--- Project.toml | 4 ++-- 2 files changed, 5 insertions(+), 5 deletions(-) diff --git a/.github/workflows/ci.yml b/.github/workflows/ci.yml index 529fb4d..b9820f6 100644 --- a/.github/workflows/ci.yml +++ b/.github/workflows/ci.yml @@ -10,7 +10,7 @@ jobs: name: Julia ${{ matrix.version }} - ${{ matrix.os }} - ${{ matrix.arch }} - ${{ github.event_name }} runs-on: ${{ matrix.os }} env: - P4EST_ROOT_DIR: "/opt/p4est/2.2/" + P4EST_ROOT_DIR: "/opt/p4est/2.3.6/" strategy: fail-fast: false matrix: @@ -43,10 +43,10 @@ jobs: - name: Install p4est if: steps.cache-p4est.outputs.cache-hit != 'true' run: | - # Install p4est 2.2 from sources + # Install p4est 2.3.6 from sources CURR_DIR=$(pwd) PACKAGE=p4est - VERSION=2.2 + VERSION=2.3.6 INSTALL_ROOT=/opt P4EST_INSTALL=$INSTALL_ROOT/$PACKAGE/$VERSION TAR_FILE=$PACKAGE-$VERSION.tar.gz diff --git a/Project.toml b/Project.toml index 08108fe..00ba2a1 100644 --- a/Project.toml +++ b/Project.toml @@ -1,7 +1,7 @@ name = "GridapP4est" uuid = "c2c8e14b-f5fd-423d-9666-1dd9ad120af9" authors = ["Alberto F. Martin "] -version = "0.3.5" +version = "0.3.6" [deps] ArgParse = "c7e460c6-2fb9-53a9-8c5b-16f535851c63" @@ -20,7 +20,7 @@ FillArrays = "0.8.4, 0.9, 0.10, 0.11, 0.12, 1" Gridap = "0.17.22" GridapDistributed = "0.3.1" MPI = "0.20" -P4est_wrapper = "0.2.0" +P4est_wrapper = "0.2.2" PartitionedArrays = "0.3.3" julia = "1.5,1.6,1.7,1.8,1.9" From 090e6e51d911d9fa745b3c87528ae49b40342ead Mon Sep 17 00:00:00 2001 From: Jordi Manyer Fuertes Date: Mon, 15 Apr 2024 12:28:32 +1000 Subject: [PATCH 28/29] Bump Version --- Project.toml | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/Project.toml b/Project.toml index edf8d4a..10260d6 100644 --- a/Project.toml +++ b/Project.toml @@ -1,7 +1,7 @@ name = "GridapP4est" uuid = "c2c8e14b-f5fd-423d-9666-1dd9ad120af9" authors = ["Alberto F. Martin "] -version = "0.3.6" +version = "0.3.7" [deps] ArgParse = "c7e460c6-2fb9-53a9-8c5b-16f535851c63" From 75efb3eb65e36dc5bd4d53ce00e385090c27b555 Mon Sep 17 00:00:00 2001 From: Jordi Manyer Fuertes Date: Mon, 15 Apr 2024 12:29:03 +1000 Subject: [PATCH 29/29] Minor --- Project.toml | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/Project.toml b/Project.toml index 10260d6..edf8d4a 100644 --- a/Project.toml +++ b/Project.toml @@ -1,7 +1,7 @@ name = "GridapP4est" uuid = "c2c8e14b-f5fd-423d-9666-1dd9ad120af9" authors = ["Alberto F. Martin "] -version = "0.3.7" +version = "0.3.6" [deps] ArgParse = "c7e460c6-2fb9-53a9-8c5b-16f535851c63"