Skip to content

Commit

Permalink
Made gather a new format callable
Browse files Browse the repository at this point in the history
  • Loading branch information
@SadiinsoSnowfall
SadiinsoSnowfall authored Feb 12, 2025
1 parent 0762239 commit e85f518
Show file tree
Hide file tree
Showing 6 changed files with 265 additions and 252 deletions.
101 changes: 65 additions & 36 deletions include/eve/module/core/regular/gather.hpp
View file
Original file line number Diff line number Diff line change
@@ -1,3 +1,4 @@
//==================================================================================================
/*
EVE - Expressive Vector Engine
Copyright : EVE Project Contributors
Expand All @@ -10,42 +11,70 @@

namespace eve
{
// TODO DOC
//================================================================================================
//! @addtogroup core_simd
//! @{
//! @var gather
//! @brief Computes the TODO
//!
//! @groupheader{Header file}
//!
//! @code
//! #include <eve/module/core.hpp>
//! @endcode
//!
//! @groupheader{Callable Signatures}
//!
//! @code
//! namespace eve
//! {
//! TODO
//! }
//! @endcode
//!
//! **Parameters**
//!
//! * `x`: An instance of an [SIMD value](@ref eve::simd_value)
//!
//! **Return value**
//!
//! * TODO
//!
//! @groupheader{Example}
//!
//! TODO
//! @}
//================================================================================================
EVE_MAKE_CALLABLE(gather_, gather);
template<typename Options>
struct gather_t : callable<gather_t, Options, conditional_option>
{
template<arithmetic_value T, integral_value U>
constexpr EVE_FORCEINLINE as_wide_as_t<T, U> operator()(T const* ptr, U idx) const noexcept
{
static_assert(simd_value<U> || !Options::contains(condition_key), "[eve::gather] Scalar values can't be masked by SIMD logicals");
return EVE_DISPATCH_CALL(ptr, idx);
}

template<arithmetic_value T, integral_value U, typename N>
constexpr EVE_FORCEINLINE as_wide_as_t<T, U> operator()(aligned_ptr<T, N> ptr, U idx) const noexcept
{
static_assert(simd_value<U> || !Options::contains(condition_key), "[eve::gather] Scalar values can't be masked by SIMD logicals");
return EVE_DISPATCH_CALL(ptr, idx);
}

EVE_CALLABLE_OBJECT(gather_t, gather_);
};

//================================================================================================
//! @addtogroup core_simd
//! @{
//! @var gather
//! @brief Load a SIMD value with values selected from a memory region at the given offsets.
//!
//! @groupheader{Header file}
//!
//! @code
//! #include <eve/module/core.hpp>
//! @endcode
//!
//! @groupheader{Callable Signatures}
//!
//! @code
//! namespace eve
//! {
//! template<arithmetic_value T, integral_value U>
//! as_wide_as_t<T, U> gather(T const* ptr, U idx);
//!
//! template<arithmetic_value T, integral_value U, typename N>
//! auto gather(aligned_ptr<T, N> ptr, U idx);
//! }
//! @endcode
//!
//! **Parameters**
//!
//! * `idx`: An instance of an [integral value](@ref eve::integral_value).
//! * `ptr`: A pointer to the memory region to load from.
//!
//! **Return value**
//!
//! * A [value](@ref eve::value) equivalent to:
//! @code
//! as_wide_as_t<T, U> res = { ptr[idx[0]], ptr[idx[1]], ..., ptr[idx[N-1]] };
//! @endcode
//!
//! @groupheader{Example}
//! @godbolt{doc/core/gather.cpp}
//================================================================================================
inline constexpr auto gather = functor<gather_t>;
//================================================================================================
//! @}
//================================================================================================
}

#include <eve/module/core/regular/impl/gather.hpp>
Expand Down
107 changes: 41 additions & 66 deletions include/eve/module/core/regular/impl/gather.hpp
View file
Original file line number Diff line number Diff line change
Expand Up @@ -12,78 +12,53 @@
#include <eve/detail/implementation.hpp>
#include <eve/memory/aligned_ptr.hpp>
#include <eve/module/core/constant/zero.hpp>
#include <eve/module/core/regular/if_else.hpp>

namespace eve::detail
{
//================================================================================================
// Unaligned pointer
template<typename U, integral_scalar_value T, typename N>
EVE_FORCEINLINE auto
gather_(EVE_SUPPORTS(cpu_), U const *ptr, wide<T, N> const& v) noexcept
{
wide<U, N> that = [=](auto i, auto) { return ptr[v.get(i)]; };
return that;
}
//================================================================================================
// Unaligned pointer
template<callable_options O, typename U, integral_scalar_value T, typename N>
EVE_FORCEINLINE auto gather_(EVE_REQUIRES(cpu_), O const& opts, U const *ptr, wide<T, N> v) noexcept
{
auto cx = opts[condition_key];
using C = decltype(cx);

template<typename U, conditional_expr C, integral_scalar_value T, typename N>
EVE_FORCEINLINE auto
gather_(EVE_SUPPORTS(cpu_), C cx, U const *ptr, wide<T, N> const& v) noexcept
{
auto src = alternative(cx, wide<U, N> {}, as<wide<U, N>> {});
auto m = expand_mask(cx, as<wide<U, N>> {});
if constexpr (C::is_complete && C::is_inverted)
{
return wide<U, N>{ [=](auto i, auto) { return ptr[v.get(i)]; } };
}
else
{
auto src = alternative(cx, wide<U, N>{}, as<wide<U, N>>{});
auto m = expand_mask(cx, as<wide<U, N>>{});

wide<U, N> that = [=](auto i, auto) { return m.get(i) ? ptr[v.get(i)] : src.get(i); };
return that;
}
return wide<U, N>{ [=](auto i, auto) { return m.get(i) ? ptr[v.get(i)] : src.get(i); } };
}
}

//================================================================================================
// Aligned pointer
template<typename U, typename S, integral_scalar_value T, typename N>
EVE_FORCEINLINE auto
gather_(EVE_SUPPORTS(cpu_), aligned_ptr<U, S> ptr, wide<T, N> const& v) noexcept
{
return gather(ptr.get(), v);
}
//================================================================================================
// Aligned pointer
template<callable_options O, typename U, typename S, integral_scalar_value T, typename N>
EVE_FORCEINLINE auto gather_(EVE_REQUIRES(cpu_), O const& opts, aligned_ptr<U, S> ptr, wide<T, N> v) noexcept
{
return gather.behavior(current_api, opts, ptr.get(), v);
}

template<typename U, typename S, conditional_expr C, integral_scalar_value T, typename N>
EVE_FORCEINLINE auto
gather_(EVE_SUPPORTS(cpu_), C cx, aligned_ptr<U, S> ptr, wide<T, N> const& v) noexcept
{
return gather[cx](ptr.get(), v);
}
//////////////////////////////////////////////////////////////////////////////////////////////////
// scalar index for genericity
//================================================================================================
// Unaligned pointer
template<callable_options O, typename U, integral_scalar_value T>
EVE_FORCEINLINE auto gather_(EVE_REQUIRES(cpu_), O const&, U const* ptr, T v) noexcept
{
return *(ptr + v);
}

//////////////////////////////////////////////////////////////////////////////////////////////////
// scalar index for genericity
//================================================================================================
// Unaligned pointer
template<typename U, integral_scalar_value T>
EVE_FORCEINLINE auto
gather_(EVE_SUPPORTS(cpu_), U const *ptr, T const& v) noexcept
{
return *(ptr + v);
}

template<typename U, value X, integral_scalar_value T>
EVE_FORCEINLINE auto
gather_(EVE_SUPPORTS(cpu_), logical<X> const& cond, U const *ptr, T const& v) noexcept
{
return cond ? *(ptr + v) : zero(as<U>());
}

//================================================================================================
// Aligned pointer
template<typename U, typename S, integral_scalar_value T>
EVE_FORCEINLINE auto
gather_(EVE_SUPPORTS(cpu_), aligned_ptr<U, S> ptr, T const& v) noexcept
{
return *(ptr + v);
}

template<typename U, typename S, value X, integral_scalar_value T>
EVE_FORCEINLINE auto
gather_(EVE_SUPPORTS(cpu_), logical<X> const& cond, aligned_ptr<U, S> ptr, T const& v) noexcept
{
return cond ? *(ptr + v) : zero(as<U>());
}
//================================================================================================
// Aligned pointer
template<callable_options O, typename U, typename S, integral_scalar_value T>
EVE_FORCEINLINE auto gather_(EVE_REQUIRES(cpu_), O const&, aligned_ptr<U, S> ptr, T v) noexcept
{
return *(ptr.get() + v);
}
}
62 changes: 28 additions & 34 deletions include/eve/module/core/regular/impl/simd/arm/sve/gather.hpp
View file
Original file line number Diff line number Diff line change
Expand Up @@ -16,43 +16,37 @@

namespace eve::detail
{
template<typename U, integral_scalar_value T, typename N>
EVE_FORCEINLINE auto
gather_(EVE_SUPPORTS(sve_), U const* p, wide<T, N> v) noexcept -> wide<U, N>
requires sve_abi<abi_t<T, N>>
{
return gather[ignore_none](p,v);
}
template<callable_options O, typename U, integral_scalar_value T, typename N>
EVE_FORCEINLINE wide<U, N> gather_(EVE_REQUIRES(sve_), O const& opts, U const* p, wide<T, N> v) noexcept
requires sve_abi<abi_t<T, N>>
{
using out_t = wide<U, N>;
using u_t = make_integer_t<sizeof(U)>;

template<typename U, conditional_expr C, integral_scalar_value T, typename N>
EVE_FORCEINLINE auto
gather_(EVE_SUPPORTS(sve_), C cx, U const* p, wide<T, N> v) noexcept -> wide<U, N>
requires sve_abi<abi_t<T, N>>
{
using out_t = wide<U, N>;
using u_t = make_integer_t<sizeof(U)>;
auto cx = opts[condition_key];
using C = decltype(cx);

// Ignore All case : just return the alternative if any
if constexpr(C::is_complete && !C::is_inverted) return alternative(cx, out_t{}, as<out_t>{});
// Aggregation cases
else if constexpr(has_aggregated_abi_v<out_t>) return gather_(EVE_RETARGET(cpu_),cx,p,v);
// Smaller data goes through the generic cases
else if constexpr(sizeof(U) <= 2) return gather_(EVE_RETARGET(cpu_),cx,p,v);
// Small index get converted then we recall gather
else if constexpr(sizeof(T) < 4) return gather[cx](p, convert(v, as<u_t>{}));
// SVE gather requires same size for index and values
else if constexpr(sizeof(T) != sizeof(U)) return gather[cx](p, convert(v, as<u_t>{}));
else
{
auto s = alternative(cx, out_t{}, as<out_t> {});
auto m = expand_mask(cx, as<out_t> {});
// Ignore All case : just return the alternative if any
if constexpr (C::is_complete && !C::is_inverted) return alternative(cx, out_t{}, as<out_t>{});
// Aggregation cases
else if constexpr (has_aggregated_abi_v<out_t>) return gather.behavior(cpu_{}, opts, p, v);
// Smaller data goes through the generic cases
else if constexpr (sizeof(U) <= 2) return gather.behavior(cpu_{}, opts, p, v);
// Small index get converted then we recall gather
else if constexpr (sizeof(T) < 4) return gather[cx](p, convert(v, as<u_t>{}));
// SVE gather requires same size for index and values
else if constexpr (sizeof(T) != sizeof(U)) return gather[cx](p, convert(v, as<u_t>{}));
else
{
auto s = alternative(cx, out_t{}, as<out_t> {});
auto m = expand_mask(cx, as<out_t> {});

// Fix up offset
v *= sizeof(U);
out_t res = svld1_gather_offset(m, p, v);
// Fix up offset
v *= sizeof(U);
out_t res = svld1_gather_offset(m, p, v);

if constexpr(C::has_alternative) return eve::replace_ignored(res, cx, s);
else return res;
if constexpr(C::has_alternative) return eve::replace_ignored(res, cx, s);
else return res;
}
}
}
}
Loading

0 comments on commit e85f518

Please sign in to comment.