-
Notifications
You must be signed in to change notification settings - Fork 58
Commit
This commit does not belong to any branch on this repository, and may belong to a fork outside of the repository.
- Loading branch information
1 parent
d2ec2f6
commit 7a889e6
Showing
14 changed files
with
662 additions
and
2 deletions.
There are no files selected for viewing
This file contains bidirectional Unicode text that may be interpreted or compiled differently than what appears below. To review, open the file in an editor that reveals hidden Unicode characters.
Learn more about bidirectional Unicode characters
This file contains bidirectional Unicode text that may be interpreted or compiled differently than what appears below. To review, open the file in an editor that reveals hidden Unicode characters.
Learn more about bidirectional Unicode characters
This file contains bidirectional Unicode text that may be interpreted or compiled differently than what appears below. To review, open the file in an editor that reveals hidden Unicode characters.
Learn more about bidirectional Unicode characters
This file contains bidirectional Unicode text that may be interpreted or compiled differently than what appears below. To review, open the file in an editor that reveals hidden Unicode characters.
Learn more about bidirectional Unicode characters
This file contains bidirectional Unicode text that may be interpreted or compiled differently than what appears below. To review, open the file in an editor that reveals hidden Unicode characters.
Learn more about bidirectional Unicode characters
| Original file line number | Diff line number | Diff line change |
|---|---|---|
| @@ -0,0 +1,129 @@ | ||
| //================================================================================================== | ||
| /* | ||
| EVE - Expressive Vector Engine | ||
| Copyright : EVE Project Contributors | ||
| SPDX-License-Identifier: BSL-1.0 | ||
| */ | ||
| //================================================================================================== | ||
| #pragma once | ||
|
|
||
| #include <eve/module/algo/algo/array_utils.hpp> | ||
| #include <eve/module/algo/algo/common_forceinline_lambdas.hpp> | ||
| #include <eve/module/algo/algo/for_each_iteration.hpp> | ||
| #include <eve/module/algo/algo/preprocess_range.hpp> | ||
| #include <eve/module/algo/algo/traits.hpp> | ||
| #include <eve/module/core.hpp> | ||
|
|
||
| namespace eve::algo | ||
| { | ||
|
|
||
| template<typename TraitsSupport> struct for_each_selected_ : TraitsSupport | ||
| { | ||
| template<typename LoopBody, typename I> struct LoopBodyAdapter | ||
| { | ||
| LoopBody& loop_body; | ||
| I base; | ||
|
|
||
| EVE_FORCEINLINE | ||
| bool operator()(std::ptrdiff_t i) { return loop_body(base + i); } | ||
| }; | ||
|
|
||
| template<typename P, typename LoopBody, typename ProcessedI, typename I> struct delegate | ||
| { | ||
| P is_selected; | ||
| LoopBody& loop_body; | ||
| bool was_stopped; | ||
| ProcessedI processed_first; | ||
| I first; | ||
|
|
||
| EVE_FORCEINLINE I restore_iterator(const auto& it) { return first + (it - processed_first); } | ||
|
|
||
| EVE_FORCEINLINE bool step(auto it, eve::relative_conditional_expr auto ignore, auto /*idx*/) | ||
| { | ||
| auto loaded = eve::load[ignore](it); | ||
| was_stopped = eve::iterate_selected[ignore]( | ||
| is_selected(loaded), LoopBodyAdapter<LoopBody, I> {loop_body, restore_iterator(it)}); | ||
| return was_stopped; | ||
| } | ||
|
|
||
| EVE_FORCEINLINE bool unrolled_step(auto arr) | ||
| { | ||
| return unroll_by_calling_single_step {}(arr, *this); | ||
| } | ||
| }; | ||
|
|
||
| template<relaxed_range Rng, typename P, irregular_predicate<unaligned_iterator_t<Rng>> LoopBody> | ||
| EVE_FORCEINLINE bool operator()(Rng&& rng, P is_selected, LoopBody&& loop_body) const | ||
| { | ||
| if( rng.begin() == rng.end() ) return false; | ||
| auto processed = preprocess_range(TraitsSupport::get_traits(), EVE_FWD(rng)); | ||
|
|
||
| auto iteration = | ||
| algo::for_each_iteration(processed.traits(), processed.begin(), processed.end()); | ||
|
|
||
| using ProcessedI = decltype(processed.begin()); | ||
| using I = unaligned_iterator_t<Rng>; | ||
| delegate<P, LoopBody, ProcessedI, I> d { | ||
| is_selected, loop_body, false, processed.begin(), unalign(rng.begin())}; | ||
| iteration(d); | ||
| return d.was_stopped; | ||
| } | ||
| }; | ||
|
|
||
| //================================================================================================ | ||
| //! @addtogroup algos | ||
| //! @{ | ||
| //! @var for_each_selected | ||
| //! | ||
| //! @brief an algorithm to perform some scalar operation for every element, that matches a | ||
| //! given SIMD predicate. | ||
| //! **Defined in Header** | ||
| //! | ||
| //! @code | ||
| //! #include <eve/module/algo.hpp> | ||
| //! @endcode | ||
| //! | ||
| //! The scalar equivalent of this algorithm would be: | ||
| //! | ||
| //! @code | ||
| //! bool for_each_selected(std::ranges::forward_range auto&& r, | ||
| //! auto is_selected, | ||
| //! auto&& loop_body) { | ||
| //! for (auto f = std::ranges::begin(r); f != std::ranges::end(r); ++f) { | ||
| //! if (is_selected(*f) && loop_body(f)) { | ||
| //! return true; | ||
| //! } | ||
| //! } | ||
| //! return false; | ||
| //! } | ||
| //! @endcode | ||
| //! | ||
| //! @groupheader{Callable Signatures} | ||
| //! | ||
| //! @code | ||
| //! namespace eve::algo | ||
| //! { | ||
| //! template<relaxed_range Rng, typename P, irregular_predicate<unaligned_iterator_t<Rng>> LoopBody> | ||
| //! bool for_each_selected(Rng&& rng, P is_selected, LoopBody loop_body); | ||
| //! } | ||
| //! @endcode | ||
| //! | ||
| //! **Parameters** | ||
| //! | ||
| //! * rng - relaxed range which we iterate | ||
| //! * is_selected - simd predicate to check values | ||
| //! * loop_body - operation to perform for each value (should return true to break), | ||
| //! | ||
| //! **Return value** | ||
| //! | ||
| //! returns true iff the iteration was interrupted. | ||
| //! | ||
| //! @groupheader{Example} | ||
| //! | ||
| //! @godbolt{doc/algo/for_each_selected.cpp} | ||
| //! @} | ||
| //================================================================================================ | ||
|
|
||
| inline constexpr auto for_each_selected = function_with_traits<for_each_selected_>; | ||
|
|
||
| } |
This file contains bidirectional Unicode text that may be interpreted or compiled differently than what appears below. To review, open the file in an editor that reveals hidden Unicode characters.
Learn more about bidirectional Unicode characters
This file contains bidirectional Unicode text that may be interpreted or compiled differently than what appears below. To review, open the file in an editor that reveals hidden Unicode characters.
Learn more about bidirectional Unicode characters
| Original file line number | Diff line number | Diff line change |
|---|---|---|
| @@ -0,0 +1,89 @@ | ||
| //================================================================================================== | ||
| /* | ||
| EVE - Expressive Vector Engine | ||
| Copyright : EVE Project Contributors | ||
| SPDX-License-Identifier: BSL-1.0 | ||
| */ | ||
| //================================================================================================== | ||
| #pragma once | ||
|
|
||
| #include <eve/arch/top_bits.hpp> | ||
| #include <eve/conditional.hpp> | ||
|
|
||
| #include <bit> | ||
|
|
||
| namespace eve::detail | ||
| { | ||
|
|
||
| template<typename F> struct plus_offset_lambda | ||
| { | ||
| F& f; | ||
| std::ptrdiff_t offset; | ||
|
|
||
| EVE_FORCEINLINE | ||
| bool operator()(std::ptrdiff_t i) const { return f(i + offset); } | ||
| }; | ||
|
|
||
| template<std::ptrdiff_t bits_per_element, std::unsigned_integral Bits> | ||
| EVE_FORCEINLINE bool | ||
| iterate_selected_int(Bits bits, auto&& f) | ||
| { | ||
| if constexpr( bits_per_element <= 2 ) | ||
| { | ||
| while( bits ) | ||
| { | ||
| std::ptrdiff_t i = std::countr_zero(bits); | ||
| if( f(i / bits_per_element) ) { return true; } | ||
|
|
||
| bits = bits & (bits - 1); | ||
| if( bits_per_element == 2 ) bits = bits & (bits - 1); | ||
| } | ||
| return false; | ||
| } | ||
| else | ||
| { | ||
| std::ptrdiff_t offset = 0; | ||
| while( bits ) | ||
| { | ||
| std::ptrdiff_t count = std::countr_zero(bits); | ||
| offset += count; | ||
| if( f(offset / bits_per_element) ) { return true; } | ||
| offset += bits_per_element; | ||
| bits >>= count; | ||
| bits >>= bits_per_element; | ||
| } | ||
| return false; | ||
| } | ||
| } | ||
|
|
||
| template<callable_options O, typename L, typename F> | ||
| EVE_FORCEINLINE bool | ||
| iterate_selected_(EVE_REQUIRES(cpu_), O const& opts, L l, F&& f) | ||
| { | ||
| auto ignore = opts[condition_key]; | ||
| if constexpr( std::same_as<L, bool> ) | ||
| { | ||
| if( l && ignore.count(eve::as<eve::wide<int, eve::fixed<1>>> {}) ) | ||
| { | ||
| return f(std::ptrdiff_t {0}); | ||
| } | ||
| return false; | ||
| } | ||
| else if constexpr( L::size() == 1U ) { return iterate_selected[opts](l.get(0), f); } | ||
| else if constexpr( logical_simd_value<L> ) | ||
| { | ||
| return iterate_selected[ignore_none](top_bits {l, ignore}, f); | ||
| } | ||
| else if constexpr( !std::same_as<decltype(ignore), ignore_none_> ) | ||
| { | ||
| return iterate_selected[ignore_none](l & L {ignore}, f); | ||
| } | ||
| else if constexpr( L::is_aggregated ) | ||
| { | ||
| auto [lo, hi] = l.slice(); | ||
| if( iterate_selected(lo, f) ) { return true; } | ||
| return iterate_selected(hi, plus_offset_lambda<F> {f, L::size() / 2}); | ||
| } | ||
| else { return iterate_selected_int<L::bits_per_element>(l.as_int(), f); } | ||
| } | ||
| } |
41 changes: 41 additions & 0 deletions
41
include/eve/module/core/regular/impl/simd/arm/sve/iterate_selected.hpp
This file contains bidirectional Unicode text that may be interpreted or compiled differently than what appears below. To review, open the file in an editor that reveals hidden Unicode characters.
Learn more about bidirectional Unicode characters
| Original file line number | Diff line number | Diff line change |
|---|---|---|
| @@ -0,0 +1,41 @@ | ||
| //================================================================================================== | ||
| /* | ||
| EVE - Expressive Vector Engine | ||
| Copyright : EVE Project Contributors | ||
| SPDX-License-Identifier: BSL-1.0 | ||
| */ | ||
| //================================================================================================== | ||
| #pragma once | ||
|
|
||
| #include <eve/module/core/regular/any.hpp> | ||
| #include <eve/module/core/regular/count_true.hpp> | ||
|
|
||
| namespace eve::detail | ||
| { | ||
|
|
||
| template<callable_options O, typename L, typename F> | ||
| EVE_FORCEINLINE bool | ||
| iterate_selected_(EVE_REQUIRES(sve_), O const& opts, top_bits<L> l, F&& f) | ||
| requires(L::size() > 1 && !top_bits<L>::is_aggregated) | ||
| { | ||
| auto ignore = opts[condition_key]; | ||
| if constexpr( !std::same_as<decltype(ignore), ignore_none_> ) | ||
| { | ||
| return iterate_selected[ignore_none](l & top_bits<L>{ignore}, f); | ||
| } | ||
| else | ||
| { | ||
| auto m = l.storage; | ||
| while( any(m) ) | ||
| { | ||
| L ignored_before = svbrka_z(sve_true<element_type_t<L>>(), m); | ||
| m = svbic_z(sve_true<element_type_t<L>>(), m, ignored_before); | ||
| if (f(count_true(ignored_before) - 1)) { | ||
| return true; | ||
| } | ||
| } | ||
| return false; | ||
| } | ||
| } | ||
|
|
||
| } |
Oops, something went wrong.