Auto vectorize replace_copy, replace_copy_if (#4431)

Co-authored-by: Stephan T. Lavavej <stl@nuwen.net>
microsoft · Mar 21, 2024 · 1e7d7f8 · 1e7d7f8
1 parent acb1178
commit 1e7d7f8
Show file tree

Hide file tree

Showing 3 changed files with 109 additions and 12 deletions.
diff --git a/benchmarks/CMakeLists.txt b/benchmarks/CMakeLists.txt
@@ -115,6 +115,7 @@ add_benchmark(minmax_element src/minmax_element.cpp)
 add_benchmark(path_lexically_normal src/path_lexically_normal.cpp)
 add_benchmark(priority_queue_push_range src/priority_queue_push_range.cpp)
 add_benchmark(random_integer_generation src/random_integer_generation.cpp)
+add_benchmark(replace src/replace.cpp)
 add_benchmark(std_copy src/std_copy.cpp)
 
 add_benchmark(vector_bool_copy src/std/containers/sequences/vector.bool/copy/test.cpp)

diff --git a/benchmarks/src/replace.cpp b/benchmarks/src/replace.cpp
@@ -0,0 +1,71 @@
+// Copyright (c) Microsoft Corporation.
+// SPDX-License-Identifier: Apache-2.0 WITH LLVM-exception
+
+#include <algorithm>
+#include <benchmark/benchmark.h>
+#include <cstdint>
+#include <vector>
+
+const char src[] =
+    "Lorem ipsum dolor sit amet, consectetur adipiscing elit. Nullam mollis imperdiet massa, at dapibus elit interdum "
+    "ac. In eget sollicitudin mi. Nam at tellus at sapien tincidunt sollicitudin vel non eros. Pellentesque nunc nunc, "
+    "ullamcorper eu accumsan at, pulvinar non turpis. Quisque vel mauris pulvinar, pretium purus vel, ultricies erat. "
+    "Curabitur a magna in ligula tristique ornare. Quisque commodo, massa viverra laoreet luctus, sem nisi aliquet "
+    "velit, fermentum pulvinar velit leo eget justo. Suspendisse vel erat efficitur, pulvinar eros volutpat, vulputate "
+    "ex. Phasellus non purus vel velit tristique tristique id at ligula. Quisque mollis sodales magna. Mauris et quam "
+    "eu quam viverra tempus. Nullam tempus maximus porta. Nunc mattis eleifend fermentum. Nullam aliquam libero "
+    "accumsan velit elementum, eu laoreet metus convallis. Donec pellentesque lacus ut iaculis iaculis. Curabitur orci "
+    "elit, bibendum sit amet feugiat at, iaculis sit amet massa. Maecenas imperdiet lacus at vehicula iaculis. Donec "
+    "volutpat nunc sit amet accumsan tempor. Quisque pretium vestibulum ultricies. Suspendisse potenti. Aenean at diam "
+    "iaculis, condimentum felis venenatis, condimentum erat. Nam quis elit dui. Duis quis odio vitae metus hendrerit "
+    "rhoncus ut et magna. Cras ac augue quis nibh pharetra sagittis. Donec ullamcorper vel eros semper pretium. Proin "
+    "vel sollicitudin eros. Nulla sollicitudin mattis turpis id suscipit. Aliquam sed risus velit. Aliquam iaculis nec "
+    "nibh ac egestas. Duis finibus semper est sed consequat. Sed in sapien quis nibh dignissim mattis. Vestibulum nec "
+    "metus sodales, euismod mauris ac, sollicitudin libero. Maecenas non arcu ac velit ullamcorper fringilla et quis "
+    "nulla. Curabitur posuere leo eget ipsum tincidunt dignissim. Cras ultricies suscipit neque, quis suscipit tortor "
+    "venenatis non. Cras nisl mi, bibendum in vulputate quis, vestibulum ornare enim. Nunc hendrerit placerat dui, "
+    "aliquam mollis sem convallis et. Integer vitae urna diam. Phasellus et imperdiet est. Maecenas auctor facilisis "
+    "nibh non commodo. Suspendisse iaculis quam id bibendum feugiat. Pellentesque felis erat, egestas a libero ac, "
+    "laoreet consectetur elit. Cras ut suscipit ex. Etiam gravida sem quis ex porta, eu lacinia tortor fermentum. "
+    "Nulla consequat odio enim, sed condimentum est sagittis a. Quisque nec commodo tellus. Phasellus elementum "
+    "feugiat dolor et feugiat. Praesent sed mattis tortor. In vitae sodales purus. Morbi accumsan, ligula et interdum "
+    "lacinia, leo risus suscipit urna, non luctus mi justo eu ipsum. Curabitur venenatis pretium orci id porttitor. "
+    "Quisque dapibus nisl sit amet elit lobortis sagittis. Orci varius natoque penatibus et magnis dis parturient "
+    "montes, nascetur ridiculus mus. Mauris varius dui sit amet tortor facilisis vestibulum. Curabitur condimentum "
+    "justo nec orci mattis auctor. Quisque aliquet condimentum arcu ac sollicitudin. Maecenas elit elit, condimentum "
+    "vitae auctor a, cursus et sem. Cras vehicula ante in consequat fermentum. Praesent at massa nisi. Mauris pretium "
+    "euismod eros, ut posuere ligula ullamcorper id. Nullam aliquam malesuada est at dignissim. Pellentesque finibus "
+    "sagittis libero nec bibendum. Phasellus dolor ipsum, finibus quis turpis quis, mollis interdum felis.";
+
+template <class T>
+void rc(benchmark::State& state) {
+    const std::vector<T> a(std::begin(src), std::end(src));
+    std::vector<T> b(std::size(src));
+
+    for (auto _ : state) {
+        std::replace_copy(std::begin(a), std::end(a), std::begin(b), T{'m'}, T{'w'});
+    }
+}
+
+template <class T>
+void rc_if(benchmark::State& state) {
+    const std::vector<T> a(std::begin(src), std::end(src));
+    std::vector<T> b(std::size(src));
+
+    for (auto _ : state) {
+        (void) std::replace_copy_if(
+            std::begin(a), std::end(a), std::begin(b), [](auto x) { return x <= T{'Z'}; }, T{'X'});
+    }
+}
+
+BENCHMARK(rc<std::uint8_t>);
+BENCHMARK(rc<std::uint16_t>);
+BENCHMARK(rc<std::uint32_t>);
+BENCHMARK(rc<std::uint64_t>);
+
+BENCHMARK(rc_if<std::uint8_t>);
+BENCHMARK(rc_if<std::uint16_t>);
+BENCHMARK(rc_if<std::uint32_t>);
+BENCHMARK(rc_if<std::uint64_t>);
+
+BENCHMARK_MAIN();
diff --git a/stl/inc/algorithm b/stl/inc/algorithm
@@ -3843,6 +3843,15 @@ namespace ranges {
 } // namespace ranges
 #endif // _HAS_CXX20
 
+// TRANSITION, DevCom-10606350: help the compiler auto-vectorize for simple types
+template <class _UOutIt, class _InTy, class _NewTy, class _OutTy = remove_pointer_t<_UOutIt>>
+_INLINE_VAR constexpr bool _Can_vectorize_replace_copy = conjunction_v<is_pointer<_UOutIt>, is_same<_InTy, _NewTy>,
+    disjunction<
+#ifdef __cpp_lib_byte
+        conjunction<is_same<_InTy, byte>, is_same<_OutTy, byte>>,
+#endif // defined(__cpp_lib_byte)
+        conjunction<is_integral<_InTy>, is_integral<_OutTy>>, conjunction<is_pointer<_InTy>, is_pointer<_OutTy>>>>;
+
 _EXPORT_STD template <class _InIt, class _OutIt, class _Ty>
 _CONSTEXPR20 _OutIt replace_copy(_InIt _First, _InIt _Last, _OutIt _Dest, const _Ty& _Oldval, const _Ty& _Newval) {
     // copy replacing each matching _Oldval with _Newval
@@ -3851,10 +3860,14 @@ _CONSTEXPR20 _OutIt replace_copy(_InIt _First, _InIt _Last, _OutIt _Dest, const
     const auto _ULast = _STD _Get_unwrapped(_Last);
     auto _UDest       = _STD _Get_unwrapped_n(_Dest, _STD _Idl_distance<_InIt>(_UFirst, _ULast));
     for (; _UFirst != _ULast; ++_UFirst, (void) ++_UDest) {
-        if (*_UFirst == _Oldval) {
-            *_UDest = _Newval;
+        if constexpr (_Can_vectorize_replace_copy<decltype(_UDest), _Iter_value_t<_InIt>, _Ty>) {
+            *_UDest = *_UFirst == _Oldval ? _Newval : *_UFirst;
         } else {
-            *_UDest = *_UFirst;
+            if (*_UFirst == _Oldval) {
+                *_UDest = _Newval;
+            } else {
+                *_UDest = *_UFirst;
+            }
         }
     }
 
@@ -3921,10 +3934,14 @@ namespace ranges {
             _STL_INTERNAL_STATIC_ASSERT(indirect_binary_predicate<equal_to, projected<_It, _Pj>, const _Ty1*>);
 
             for (; _First != _Last; ++_First, (void) ++_Result) {
-                if (_STD invoke(_Proj, *_First) == _Oldval) {
-                    *_Result = _Newval;
+                if constexpr (_Can_vectorize_replace_copy<_Out, iter_value_t<_It>, _Ty2>) {
+                    *_Result = _STD invoke(_Proj, *_First) == _Oldval ? _Newval : *_First;
                 } else {
-                    *_Result = *_First;
+                    if (_STD invoke(_Proj, *_First) == _Oldval) {
+                        *_Result = _Newval;
+                    } else {
+                        *_Result = *_First;
+                    }
                 }
             }
 
@@ -3944,10 +3961,14 @@ _CONSTEXPR20 _OutIt replace_copy_if(_InIt _First, _InIt _Last, _OutIt _Dest, _Pr
     const auto _ULast = _STD _Get_unwrapped(_Last);
     auto _UDest       = _STD _Get_unwrapped_n(_Dest, _STD _Idl_distance<_InIt>(_UFirst, _ULast));
     for (; _UFirst != _ULast; ++_UFirst, (void) ++_UDest) {
-        if (_Pred(*_UFirst)) {
-            *_UDest = _Val;
+        if constexpr (_Can_vectorize_replace_copy<decltype(_UDest), _Iter_value_t<_InIt>, _Ty>) {
+            *_UDest = _Pred(*_UFirst) ? _Val : *_UFirst;
         } else {
-            *_UDest = *_UFirst;
+            if (_Pred(*_UFirst)) {
+                *_UDest = _Val;
+            } else {
+                *_UDest = *_UFirst;
+            }
         }
     }
 
@@ -4014,10 +4035,14 @@ namespace ranges {
             _STL_INTERNAL_STATIC_ASSERT(indirect_unary_predicate<_Pr, projected<_It, _Pj>>);
 
             for (; _First != _Last; ++_First, (void) ++_Result) {
-                if (_STD invoke(_Pred, _STD invoke(_Proj, *_First))) {
-                    *_Result = _Newval;
+                if constexpr (_Can_vectorize_replace_copy<_Out, iter_value_t<_It>, _Ty>) {
+                    *_Result = _STD invoke(_Pred, _STD invoke(_Proj, *_First)) ? _Newval : *_First;
                 } else {
-                    *_Result = *_First;
+                    if (_STD invoke(_Pred, _STD invoke(_Proj, *_First))) {
+                        *_Result = _Newval;
+                    } else {
+                        *_Result = *_First;
+                    }
                 }
             }