Skip to content
New issue

Have a question about this project? Sign up for a free GitHub account to open an issue and contact its maintainers and the community.

By clicking “Sign up for GitHub”, you agree to our terms of service and privacy statement. We’ll occasionally send you account related emails.

Already on GitHub? Sign in to your account

Full support for ghosted/unghosted distributed triangulations #100

Open
wants to merge 8 commits into
base: master
Choose a base branch
from
Open
Changes from 1 commit
Commits
File filter

Filter by extension

Filter by extension

Conversations
Failed to load comments.
Loading
Jump to
Jump to file
Failed to load files.
Loading
Diff view
Diff view
Next Next commit
Started refactoring the distributed code
JordiManyer committed Nov 14, 2024
commit 3185beb8c5e2af57af4bc9009d9be659fdd455b3
13 changes: 5 additions & 8 deletions src/Distributed/Distributed.jl
Original file line number Diff line number Diff line change
@@ -31,14 +31,11 @@ using Gridap.Geometry: get_face_to_parent_face
using Gridap.Arrays: find_inverse_index_map, testitem, return_type
using Gridap.FESpaces: FESpaceWithLinearConstraints
using Gridap.FESpaces: _dof_to_DOF, _DOF_to_dof
using GridapDistributed: DistributedDiscreteModel
using GridapDistributed: DistributedTriangulation
using GridapDistributed: DistributedFESpace
using GridapDistributed: DistributedSingleFieldFESpace
using GridapDistributed: DistributedMeasure
using GridapDistributed: add_ghost_cells
using GridapDistributed: generate_gids
using GridapDistributed: generate_cell_gids

using GridapDistributed: DistributedDiscreteModel, DistributedTriangulation, DistributedMeasure
using GridapDistributed: DistributedFESpace, DistributedSingleFieldFESpace
using GridapDistributed: NoGhost, WithGhost, filter_cells_when_needed, add_ghost_cells
using GridapDistributed: generate_gids, generate_cell_gids
using GridapDistributed: _find_vector_type

import GridapEmbedded.AgFEM: aggregate
114 changes: 34 additions & 80 deletions src/Distributed/DistributedDiscreteGeometries.jl
Original file line number Diff line number Diff line change
@@ -4,41 +4,9 @@ end

local_views(a::DistributedDiscreteGeometry) = a.geometries

# TODO: Is this really necessary?
function _get_values_at_owned_coords(φh,model::DistributedDiscreteModel{Dc,Dp}) where {Dc,Dp}
@assert DomainStyle(φh) == ReferenceDomain()
gids = get_cell_gids(model)
values = map(local_views(φh),local_views(model),local_views(gids)) do φh, model, gids
own_model = remove_ghost_cells(model,gids)
own_cells = get_face_to_parent_face(own_model,Dc)

trian = get_triangulation(φh)
cell_points = get_cell_points(trian)
cell_ids = get_cell_node_ids(own_model)
cell_values = φh(cell_points)

T = eltype(testitem(cell_values))
values = Vector{T}(undef,num_nodes(own_model))

cell_ids_cache = array_cache(cell_ids)
cell_values_cache = array_cache(cell_values)
for (ocell,cell) in enumerate(own_cells)
ids = getindex!(cell_ids_cache,cell_ids,ocell)
vals = getindex!(cell_values_cache,cell_values,cell)
values[ids] .= vals
end
return values
end
return values
end

function DiscreteGeometry(φh::CellField,model::DistributedDiscreteModel;name::String="")
φ_values = _get_values_at_owned_coords(φh,model)
gids = get_cell_gids(model)
geometries = map(local_views(model),local_views(gids),local_views(φ_values)) do model,gids,loc_φ
ownmodel = remove_ghost_cells(model,gids)
point_to_coords = collect1d(get_node_coordinates(ownmodel))
DiscreteGeometry(loc_φ,point_to_coords;name)
geometries = map(local_views(φh),local_views(model)) do φh, model
DiscreteGeometry(φh,model;name)
end
DistributedDiscreteGeometry(geometries)
end
@@ -48,56 +16,45 @@ function distributed_geometry(a::AbstractArray{<:DiscreteGeometry})
end

function discretize(a::AnalyticalGeometry,model::DistributedDiscreteModel)
gids = get_cell_gids(model)
geometries = map(local_views(model),local_views(gids)) do model,gids
ownmodel = remove_ghost_cells(model,gids)
point_to_coords = collect1d(get_node_coordinates(ownmodel))
discretize(a,point_to_coords)
geometries = map(local_views(model)) do model
discretize(a,model)
end
DistributedDiscreteGeometry(geometries)
end

function cut(cutter::Cutter,bgmodel::DistributedDiscreteModel,geom::DistributedDiscreteGeometry)
function cut(
cutter::Cutter,bgmodel::DistributedDiscreteModel,geom::DistributedDiscreteGeometry
)
gids = get_cell_gids(bgmodel)
cuts = map(local_views(bgmodel),local_views(gids),local_views(geom)) do bgmodel,gids,geom
ownmodel = remove_ghost_cells(bgmodel,gids)
cutgeo = cut(cutter,ownmodel,geom)
change_bgmodel(cutgeo,bgmodel,own_to_local(gids))
cuts = map(local_views(bgmodel),local_views(geom)) do bgmodel,geom
cut(cutter,bgmodel,geom)
end
consistent_bgcell_to_inoutcut!(cuts,gids)
@notimplementedif !isconsistent_bgcell_to_inoutcut(cuts,partition(gids))
DistributedEmbeddedDiscretization(cuts,bgmodel)
end

function cut_facets(cutter::Cutter,bgmodel::DistributedDiscreteModel,geom::DistributedDiscreteGeometry)
D = map(num_dims,local_views(bgmodel)) |> PartitionedArrays.getany
cell_gids = get_cell_gids(bgmodel)
facet_gids = get_face_gids(bgmodel,D-1)
cuts = map(
local_views(bgmodel),
local_views(cell_gids),
local_views(facet_gids),
local_views(geom)) do bgmodel,cell_gids,facet_gids,geom
ownmodel = remove_ghost_cells(bgmodel,cell_gids)
facet_to_pfacet = get_face_to_parent_face(ownmodel,D-1)
cutfacets = cut_facets(cutter,ownmodel,geom)
cutfacets = change_bgmodel(cutfacets,bgmodel,facet_to_pfacet)
remove_ghost_subfacets(cutfacets,facet_gids)
function cut_facets(
cutter::Cutter,bgmodel::DistributedDiscreteModel{Dc},geom::DistributedDiscreteGeometry
) where Dc
gids = get_face_gids(bgmodel,Dc-1)
cuts = map(local_views(bgmodel),local_views(geom)) do bgmodel,geom
cut_facets(cutter,bgmodel,geom)
end
consistent_bgfacet_to_inoutcut!(cuts,facet_gids)
@notimplementedif !isconsistent_bgcell_to_inoutcut(cuts,partition(gids))
DistributedEmbeddedDiscretization(cuts,bgmodel)
end

function distributed_embedded_triangulation(
T,
cutgeo::DistributedEmbeddedDiscretization,
cutinorout,
geom::DistributedDiscreteGeometry
)
trians = map(local_views(cutgeo),local_views(geom)) do lcutgeo,lgeom
T(lcutgeo,cutinorout,lgeom)
for TT in (:Triangulation,:SkeletonTriangulation,:BoundaryTriangulation,:EmbeddedBoundary)
@eval begin
function $TT(portion,cutgeo::DistributedEmbeddedDiscretization,cutinorout,geom::DistributedDiscreteGeometry)
model = get_background_model(cutgeo)
gids = get_cell_gids(model)
trians = map(local_views(cutgeo),local_views(geom),partition(gids)) do cutgeo, geom, gids
$TT(portion,gids,cutgeo,cutinorout,geom)
end
DistributedTriangulation(trians,model)
end
end
bgmodel = get_background_model(cutgeo)
DistributedTriangulation(trians,bgmodel)
end

function distributed_aggregate(
@@ -111,21 +68,18 @@ function distributed_aggregate(
_distributed_aggregate_by_threshold(strategy.threshold,cut,geo,in_or_out,facet_to_inoutcut)
end

function compute_bgcell_to_inoutcut(cutgeo::DistributedEmbeddedDiscretization,geo::DistributedDiscreteGeometry)
map(local_views(cutgeo),local_views(geo)) do cutgeo,geo
function compute_bgcell_to_inoutcut(
cutgeo::DistributedEmbeddedDiscretization,geo::DistributedDiscreteGeometry
)
map(local_views(cutgeo),local_views(geo)) do cutgeo, geo
compute_bgcell_to_inoutcut(cutgeo,geo)
end
end

function compute_bgfacet_to_inoutcut(
cutter::Cutter,
bgmodel::DistributedDiscreteModel,
geo::DistributedDiscreteGeometry
cutter::Cutter, bgmodel::DistributedDiscreteModel, geo::DistributedDiscreteGeometry
)
gids = get_cell_gids(bgmodel)
bgf_to_ioc = map(local_views(bgmodel),local_views(gids),local_views(geo)) do model,gids,geo
ownmodel = remove_ghost_cells(model,gids)
compute_bgfacet_to_inoutcut(cutter,ownmodel,geo)
map(local_views(bgmodel),local_views(geo)) do model, geo
compute_bgfacet_to_inoutcut(cutter,model,geo)
end
compute_bgfacet_to_inoutcut(bgmodel,bgf_to_ioc)
end
Loading